dkist-processing-visp 2.20.14__py3-none-any.whl → 5.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dkist_processing_visp/__init__.py +1 -0
- dkist_processing_visp/config.py +1 -0
- dkist_processing_visp/models/constants.py +61 -20
- dkist_processing_visp/models/fits_access.py +20 -0
- dkist_processing_visp/models/metric_code.py +10 -0
- dkist_processing_visp/models/parameters.py +129 -24
- dkist_processing_visp/models/tags.py +22 -1
- dkist_processing_visp/models/task_name.py +1 -0
- dkist_processing_visp/parsers/map_repeats.py +1 -0
- dkist_processing_visp/parsers/modulator_states.py +1 -0
- dkist_processing_visp/parsers/polarimeter_mode.py +4 -2
- dkist_processing_visp/parsers/raster_step.py +4 -1
- dkist_processing_visp/parsers/spectrograph_configuration.py +75 -0
- dkist_processing_visp/parsers/time.py +24 -14
- dkist_processing_visp/parsers/visp_l0_fits_access.py +19 -8
- dkist_processing_visp/parsers/visp_l1_fits_access.py +1 -0
- dkist_processing_visp/tasks/__init__.py +1 -0
- dkist_processing_visp/tasks/assemble_movie.py +1 -0
- dkist_processing_visp/tasks/background_light.py +2 -1
- dkist_processing_visp/tasks/dark.py +5 -4
- dkist_processing_visp/tasks/geometric.py +132 -20
- dkist_processing_visp/tasks/instrument_polarization.py +128 -18
- dkist_processing_visp/tasks/l1_output_data.py +203 -0
- dkist_processing_visp/tasks/lamp.py +53 -93
- dkist_processing_visp/tasks/make_movie_frames.py +8 -6
- dkist_processing_visp/tasks/mixin/beam_access.py +1 -0
- dkist_processing_visp/tasks/mixin/corrections.py +54 -4
- dkist_processing_visp/tasks/mixin/downsample.py +1 -0
- dkist_processing_visp/tasks/parse.py +50 -17
- dkist_processing_visp/tasks/quality_metrics.py +5 -4
- dkist_processing_visp/tasks/science.py +126 -46
- dkist_processing_visp/tasks/solar.py +896 -456
- dkist_processing_visp/tasks/visp_base.py +4 -3
- dkist_processing_visp/tasks/write_l1.py +38 -10
- dkist_processing_visp/tests/conftest.py +145 -47
- dkist_processing_visp/tests/header_models.py +157 -20
- dkist_processing_visp/tests/local_trial_workflows/l0_cals_only.py +21 -78
- dkist_processing_visp/tests/local_trial_workflows/l0_polcals_as_science.py +421 -0
- dkist_processing_visp/tests/local_trial_workflows/l0_solar_gain_as_science.py +387 -0
- dkist_processing_visp/tests/local_trial_workflows/l0_to_l1.py +18 -75
- dkist_processing_visp/tests/local_trial_workflows/local_trial_helpers.py +346 -14
- dkist_processing_visp/tests/test_assemble_movie.py +2 -3
- dkist_processing_visp/tests/test_assemble_quality.py +89 -4
- dkist_processing_visp/tests/test_background_light.py +51 -44
- dkist_processing_visp/tests/test_dark.py +4 -3
- dkist_processing_visp/tests/test_downsample.py +1 -0
- dkist_processing_visp/tests/test_fits_access.py +43 -0
- dkist_processing_visp/tests/test_geometric.py +45 -4
- dkist_processing_visp/tests/test_instrument_polarization.py +72 -9
- dkist_processing_visp/tests/test_lamp.py +22 -26
- dkist_processing_visp/tests/test_make_movie_frames.py +4 -4
- dkist_processing_visp/tests/test_map_repeats.py +3 -1
- dkist_processing_visp/tests/test_parameters.py +122 -21
- dkist_processing_visp/tests/test_parse.py +164 -18
- dkist_processing_visp/tests/test_quality.py +3 -4
- dkist_processing_visp/tests/test_science.py +113 -15
- dkist_processing_visp/tests/test_solar.py +318 -99
- dkist_processing_visp/tests/test_visp_constants.py +38 -8
- dkist_processing_visp/tests/test_workflows.py +1 -0
- dkist_processing_visp/tests/test_write_l1.py +22 -3
- dkist_processing_visp/workflows/__init__.py +1 -0
- dkist_processing_visp/workflows/l0_processing.py +10 -3
- dkist_processing_visp/workflows/trial_workflows.py +8 -2
- dkist_processing_visp-5.1.1.dist-info/METADATA +552 -0
- dkist_processing_visp-5.1.1.dist-info/RECORD +94 -0
- {dkist_processing_visp-2.20.14.dist-info → dkist_processing_visp-5.1.1.dist-info}/WHEEL +1 -1
- docs/conf.py +5 -1
- docs/gain_correction.rst +52 -44
- docs/science_calibration.rst +7 -0
- dkist_processing_visp/tasks/mixin/line_zones.py +0 -115
- dkist_processing_visp-2.20.14.dist-info/METADATA +0 -196
- dkist_processing_visp-2.20.14.dist-info/RECORD +0 -89
- {dkist_processing_visp-2.20.14.dist-info → dkist_processing_visp-5.1.1.dist-info}/top_level.txt +0 -0
|
@@ -4,30 +4,45 @@ from datetime import datetime
|
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
|
|
6
6
|
import asdf
|
|
7
|
+
from astropy.io import fits
|
|
8
|
+
from dkist_header_validator import spec122_validator
|
|
9
|
+
from dkist_header_validator import spec214_validator
|
|
10
|
+
from dkist_processing_common.codecs.fits import fits_array_decoder
|
|
11
|
+
from dkist_processing_common.codecs.fits import fits_hdulist_encoder
|
|
7
12
|
from dkist_processing_common.models.constants import BudName
|
|
13
|
+
from dkist_processing_common.models.fits_access import MetadataKey
|
|
8
14
|
from dkist_processing_common.models.task_name import TaskName
|
|
9
15
|
from dkist_processing_common.parsers.cs_step import CSStepFlower
|
|
10
16
|
from dkist_processing_common.parsers.cs_step import NumCSStepBud
|
|
11
|
-
from dkist_processing_common.parsers.
|
|
17
|
+
from dkist_processing_common.parsers.retarder import RetarderNameBud
|
|
18
|
+
from dkist_processing_common.parsers.task import PolcalTaskFlower
|
|
12
19
|
from dkist_processing_common.parsers.task import TaskTypeFlower
|
|
20
|
+
from dkist_processing_common.parsers.task import parse_header_ip_task_with_gains
|
|
13
21
|
from dkist_processing_common.parsers.time import ExposureTimeFlower
|
|
14
22
|
from dkist_processing_common.parsers.time import ReadoutExpTimeFlower
|
|
15
23
|
from dkist_processing_common.parsers.time import TaskExposureTimesBud
|
|
16
24
|
from dkist_processing_common.parsers.time import TaskReadoutExpTimesBud
|
|
25
|
+
from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
|
|
26
|
+
from dkist_processing_common.parsers.unique_bud import UniqueBud
|
|
17
27
|
from dkist_processing_common.tasks import ParseL0InputDataBase
|
|
18
28
|
from dkist_processing_common.tasks import WorkflowTaskBase
|
|
19
29
|
from dkist_processing_common.tasks.mixin.globus import GlobusTransferItem
|
|
20
|
-
from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetMixin
|
|
21
30
|
from dkist_processing_common.tasks.trial_output_data import TransferTrialData
|
|
22
|
-
from
|
|
31
|
+
from dkist_processing_math.statistics import average_numpy_arrays
|
|
32
|
+
from loguru import logger
|
|
23
33
|
|
|
24
34
|
from dkist_processing_visp.models.constants import VispBudName
|
|
35
|
+
from dkist_processing_visp.models.constants import VispConstants
|
|
36
|
+
from dkist_processing_visp.models.fits_access import VispMetadataKey
|
|
37
|
+
from dkist_processing_visp.models.metric_code import VispMetricCode
|
|
25
38
|
from dkist_processing_visp.models.parameters import VispParsingParameters
|
|
26
39
|
from dkist_processing_visp.models.tags import VispTag
|
|
27
40
|
from dkist_processing_visp.models.task_name import VispTaskName
|
|
28
41
|
from dkist_processing_visp.parsers.modulator_states import ModulatorStateFlower
|
|
42
|
+
from dkist_processing_visp.parsers.spectrograph_configuration import IncidentLightAngleBud
|
|
43
|
+
from dkist_processing_visp.parsers.spectrograph_configuration import ReflectedLightAngleBud
|
|
29
44
|
from dkist_processing_visp.parsers.time import DarkReadoutExpTimePickyBud
|
|
30
|
-
from dkist_processing_visp.parsers.time import
|
|
45
|
+
from dkist_processing_visp.parsers.time import NonDarkNonPolcalTaskReadoutExpTimesBud
|
|
31
46
|
from dkist_processing_visp.parsers.visp_l0_fits_access import VispL0FitsAccess
|
|
32
47
|
from dkist_processing_visp.tasks.visp_base import VispTaskBase
|
|
33
48
|
|
|
@@ -255,7 +270,9 @@ class SaveSolarCal(SaveTaskTags):
|
|
|
255
270
|
@property
|
|
256
271
|
def tag_lists_to_save(self) -> list[list[str]]:
|
|
257
272
|
return super().tag_lists_to_save + [
|
|
258
|
-
[VispTag.quality("TASK_TYPES"), VispTag.workflow_task("SolarCalibration")]
|
|
273
|
+
[VispTag.quality("TASK_TYPES"), VispTag.workflow_task("SolarCalibration")],
|
|
274
|
+
[VispTag.quality(VispMetricCode.solar_first_vignette)],
|
|
275
|
+
[VispTag.quality(VispMetricCode.solar_final_vignette)],
|
|
259
276
|
]
|
|
260
277
|
|
|
261
278
|
@property
|
|
@@ -299,6 +316,22 @@ class LoadInstPolCal(LoadTaskTags):
|
|
|
299
316
|
return "inst_pol_cal.asdf"
|
|
300
317
|
|
|
301
318
|
|
|
319
|
+
class SaveCalibratedData(SaveTaskTags):
|
|
320
|
+
@property
|
|
321
|
+
def tag_lists_to_save(self) -> list[str]:
|
|
322
|
+
return [VispTag.frame(), VispTag.calibrated()]
|
|
323
|
+
|
|
324
|
+
@property
|
|
325
|
+
def relative_save_file(self) -> str:
|
|
326
|
+
return "calibrated_science.asdf"
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
class LoadCalibratedData(LoadTaskTags):
|
|
330
|
+
@property
|
|
331
|
+
def relative_save_file(self) -> str:
|
|
332
|
+
return "calibrated_science.asdf"
|
|
333
|
+
|
|
334
|
+
|
|
302
335
|
def set_observe_wavelength_task(wavelength: float = 630.0):
|
|
303
336
|
class SetObserveWavelength(WorkflowTaskBase):
|
|
304
337
|
def run(self):
|
|
@@ -322,6 +355,29 @@ class SetObserveExpTime(VispTaskBase):
|
|
|
322
355
|
)
|
|
323
356
|
|
|
324
357
|
|
|
358
|
+
class SetCadenceConstants(WorkflowTaskBase):
|
|
359
|
+
def run(self):
|
|
360
|
+
self.constants._update(
|
|
361
|
+
{
|
|
362
|
+
BudName.average_cadence.value: 1.0,
|
|
363
|
+
BudName.minimum_cadence.value: 0.0,
|
|
364
|
+
BudName.maximum_cadence.value: 3.0,
|
|
365
|
+
BudName.variance_cadence.value: 1,
|
|
366
|
+
}
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
class SetAxesTypes(WorkflowTaskBase):
|
|
371
|
+
def run(self):
|
|
372
|
+
self.constants._update(
|
|
373
|
+
{
|
|
374
|
+
VispBudName.axis_1_type.value: "HPLT-TAN",
|
|
375
|
+
VispBudName.axis_2_type.value: "AWAV",
|
|
376
|
+
VispBudName.axis_3_type.value: "HPLN-TAN",
|
|
377
|
+
}
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
|
|
325
381
|
class SetPolarimeterMode(VispTaskBase):
|
|
326
382
|
def run(self):
|
|
327
383
|
self.constants._update({VispBudName.polarimeter_mode.value: "observe_polarimetric"})
|
|
@@ -332,7 +388,7 @@ class SetNumModstates(VispTaskBase):
|
|
|
332
388
|
self.constants._update({BudName.num_modstates.value: 10})
|
|
333
389
|
|
|
334
390
|
|
|
335
|
-
class ParseCalOnlyL0InputData(ParseL0InputDataBase
|
|
391
|
+
class ParseCalOnlyL0InputData(ParseL0InputDataBase):
|
|
336
392
|
"""
|
|
337
393
|
Parse input ViSP data. Subclassed from the ParseL0InputDataBase task in dkist_processing_common to add ViSP specific parameters.
|
|
338
394
|
|
|
@@ -358,7 +414,7 @@ class ParseCalOnlyL0InputData(ParseL0InputDataBase, InputDatasetMixin):
|
|
|
358
414
|
workflow_name=workflow_name,
|
|
359
415
|
workflow_version=workflow_version,
|
|
360
416
|
)
|
|
361
|
-
self.parameters = VispParsingParameters(self.
|
|
417
|
+
self.parameters = VispParsingParameters(scratch=self.scratch)
|
|
362
418
|
|
|
363
419
|
@property
|
|
364
420
|
def fits_parsing_class(self):
|
|
@@ -368,38 +424,55 @@ class ParseCalOnlyL0InputData(ParseL0InputDataBase, InputDatasetMixin):
|
|
|
368
424
|
@property
|
|
369
425
|
def constant_buds(self):
|
|
370
426
|
"""Add ViSP specific constants to common constants."""
|
|
427
|
+
# TODO: Subclass ViSP parse task and *remove* unneeded things from this list
|
|
371
428
|
return super().constant_buds + [
|
|
429
|
+
UniqueBud(constant_name=VispBudName.arm_id.value, metadata_key=VispMetadataKey.arm_id),
|
|
372
430
|
NumCSStepBud(self.parameters.max_cs_step_time_sec),
|
|
373
|
-
|
|
431
|
+
NonDarkNonPolcalTaskReadoutExpTimesBud(),
|
|
374
432
|
DarkReadoutExpTimePickyBud(),
|
|
433
|
+
RetarderNameBud(),
|
|
434
|
+
IncidentLightAngleBud(),
|
|
435
|
+
ReflectedLightAngleBud(),
|
|
436
|
+
TaskUniqueBud(
|
|
437
|
+
constant_name=VispBudName.grating_constant_inverse_mm.value,
|
|
438
|
+
metadata_key=VispMetadataKey.grating_constant_inverse_mm,
|
|
439
|
+
ip_task_types=[TaskName.observe.value, TaskName.solar_gain.value],
|
|
440
|
+
task_type_parsing_function=parse_header_ip_task_with_gains,
|
|
441
|
+
),
|
|
442
|
+
TaskUniqueBud(
|
|
443
|
+
constant_name=VispBudName.solar_gain_ip_start_time.value,
|
|
444
|
+
metadata_key=MetadataKey.ip_start_time,
|
|
445
|
+
ip_task_types=TaskName.solar_gain,
|
|
446
|
+
task_type_parsing_function=parse_header_ip_task_with_gains,
|
|
447
|
+
),
|
|
375
448
|
TaskExposureTimesBud(
|
|
376
449
|
stem_name=VispBudName.lamp_exposure_times.value,
|
|
377
|
-
|
|
450
|
+
ip_task_types=TaskName.lamp_gain.value,
|
|
378
451
|
header_task_parsing_func=parse_header_ip_task_with_gains,
|
|
379
452
|
),
|
|
380
453
|
TaskExposureTimesBud(
|
|
381
454
|
stem_name=VispBudName.solar_exposure_times.value,
|
|
382
|
-
|
|
455
|
+
ip_task_types=TaskName.solar_gain.value,
|
|
383
456
|
header_task_parsing_func=parse_header_ip_task_with_gains,
|
|
384
457
|
),
|
|
385
458
|
TaskExposureTimesBud(
|
|
386
459
|
stem_name=VispBudName.polcal_exposure_times.value,
|
|
387
|
-
|
|
460
|
+
ip_task_types=TaskName.polcal.value,
|
|
388
461
|
header_task_parsing_func=parse_header_ip_task_with_gains,
|
|
389
462
|
),
|
|
390
463
|
TaskReadoutExpTimesBud(
|
|
391
464
|
stem_name=VispBudName.lamp_readout_exp_times.value,
|
|
392
|
-
|
|
465
|
+
ip_task_types=TaskName.lamp_gain.value,
|
|
393
466
|
header_task_parsing_func=parse_header_ip_task_with_gains,
|
|
394
467
|
),
|
|
395
468
|
TaskReadoutExpTimesBud(
|
|
396
469
|
stem_name=VispBudName.solar_readout_exp_times.value,
|
|
397
|
-
|
|
470
|
+
ip_task_types=TaskName.solar_gain.value,
|
|
398
471
|
header_task_parsing_func=parse_header_ip_task_with_gains,
|
|
399
472
|
),
|
|
400
473
|
TaskReadoutExpTimesBud(
|
|
401
474
|
stem_name=VispBudName.polcal_readout_exp_times.value,
|
|
402
|
-
|
|
475
|
+
ip_task_types=TaskName.polcal.value,
|
|
403
476
|
header_task_parsing_func=parse_header_ip_task_with_gains,
|
|
404
477
|
),
|
|
405
478
|
]
|
|
@@ -410,12 +483,21 @@ class ParseCalOnlyL0InputData(ParseL0InputDataBase, InputDatasetMixin):
|
|
|
410
483
|
return super().tag_flowers + [
|
|
411
484
|
CSStepFlower(max_cs_step_time_sec=self.parameters.max_cs_step_time_sec),
|
|
412
485
|
TaskTypeFlower(header_task_parsing_func=parse_header_ip_task_with_gains),
|
|
486
|
+
PolcalTaskFlower(),
|
|
413
487
|
ModulatorStateFlower(),
|
|
414
488
|
ExposureTimeFlower(),
|
|
415
489
|
ReadoutExpTimeFlower(),
|
|
416
490
|
]
|
|
417
491
|
|
|
418
492
|
|
|
493
|
+
class ValidateL1Output(VispTaskBase):
|
|
494
|
+
def run(self) -> None:
|
|
495
|
+
files = self.read(tags=[VispTag.output(), VispTag.frame()])
|
|
496
|
+
for f in files:
|
|
497
|
+
logger.info(f"Validating {f}")
|
|
498
|
+
spec214_validator.validate(f, extra=False)
|
|
499
|
+
|
|
500
|
+
|
|
419
501
|
def transfer_trial_data_locally_task(
|
|
420
502
|
trial_dir: str | Path,
|
|
421
503
|
):
|
|
@@ -443,3 +525,253 @@ def transfer_trial_data_locally_task(
|
|
|
443
525
|
os.system(f"cp {frame.source_path} {frame.destination_path}")
|
|
444
526
|
|
|
445
527
|
return LocalTrialData
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def translate_122_to_214l0_task(suffix: str):
|
|
531
|
+
class Translate122To214L0(WorkflowTaskBase):
|
|
532
|
+
def run(self) -> None:
|
|
533
|
+
raw_dir = Path(self.scratch.scratch_base_path) / f"VISP{self.recipe_run_id:03n}"
|
|
534
|
+
if not os.path.exists(self.scratch.workflow_base_path):
|
|
535
|
+
os.makedirs(self.scratch.workflow_base_path)
|
|
536
|
+
|
|
537
|
+
if not raw_dir.exists():
|
|
538
|
+
raise FileNotFoundError(
|
|
539
|
+
f"Expected to find a raw VISP{{run_id:03n}} folder in {self.scratch.scratch_base_path}"
|
|
540
|
+
)
|
|
541
|
+
|
|
542
|
+
for file in raw_dir.glob(f"*{suffix}"):
|
|
543
|
+
translated_file_name = Path(self.scratch.workflow_base_path) / os.path.basename(
|
|
544
|
+
file
|
|
545
|
+
)
|
|
546
|
+
logger.info(f"Translating {file} -> {translated_file_name}")
|
|
547
|
+
hdl = fits.open(file)
|
|
548
|
+
i = 0
|
|
549
|
+
if hdl[i].data is None:
|
|
550
|
+
i = 1
|
|
551
|
+
|
|
552
|
+
header = spec122_validator.validate_and_translate_to_214_l0(
|
|
553
|
+
hdl[i].header, return_type=fits.HDUList
|
|
554
|
+
)[0].header
|
|
555
|
+
|
|
556
|
+
comp_hdu = fits.CompImageHDU(header=header, data=hdl[i].data)
|
|
557
|
+
comp_hdl = fits.HDUList([fits.PrimaryHDU(), comp_hdu])
|
|
558
|
+
comp_hdl.writeto(translated_file_name, overwrite=True)
|
|
559
|
+
|
|
560
|
+
hdl.close()
|
|
561
|
+
del hdl
|
|
562
|
+
comp_hdl.close()
|
|
563
|
+
del comp_hdl
|
|
564
|
+
|
|
565
|
+
return Translate122To214L0
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
def tag_inputs_task(suffix: str):
|
|
569
|
+
class TagInputs(WorkflowTaskBase):
|
|
570
|
+
def run(self) -> None:
|
|
571
|
+
logger.info(f"Looking in {os.path.abspath(self.scratch.workflow_base_path)}")
|
|
572
|
+
input_file_list = list(self.scratch.workflow_base_path.glob(f"*.{suffix}"))
|
|
573
|
+
if len(input_file_list) == 0:
|
|
574
|
+
raise FileNotFoundError(
|
|
575
|
+
f"Did not find any files matching '*.{suffix}' in {self.scratch.workflow_base_path}"
|
|
576
|
+
)
|
|
577
|
+
for file in input_file_list:
|
|
578
|
+
logger.info(f"Found {file}")
|
|
579
|
+
self.tag(path=file, tags=[VispTag.input(), VispTag.frame()])
|
|
580
|
+
|
|
581
|
+
return TagInputs
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
class TagSingleSolarGainAsScience(VispTaskBase):
|
|
585
|
+
"""Do."""
|
|
586
|
+
|
|
587
|
+
def run(self) -> None:
|
|
588
|
+
"""Do."""
|
|
589
|
+
tags = [
|
|
590
|
+
VispTag.input(),
|
|
591
|
+
VispTag.frame(),
|
|
592
|
+
VispTag.task_solar_gain(),
|
|
593
|
+
]
|
|
594
|
+
file_list = list(self.read(tags=tags))
|
|
595
|
+
first_hdul = fits.open(file_list[0])
|
|
596
|
+
idx = 1 if first_hdul[0].data is None else 0
|
|
597
|
+
first_header = first_hdul[idx].header
|
|
598
|
+
logger.info(f"Averaging {len(file_list)} files")
|
|
599
|
+
arrays = self.read(tags=tags, decoder=fits_array_decoder)
|
|
600
|
+
avg_array = average_numpy_arrays(arrays=arrays)
|
|
601
|
+
|
|
602
|
+
hdul = fits.HDUList([fits.PrimaryHDU(data=avg_array, header=first_header)])
|
|
603
|
+
hdul[0].header[VispMetadataKey.raster_scan_step] = 0
|
|
604
|
+
hdul[0].header[VispMetadataKey.total_raster_steps] = 1
|
|
605
|
+
hdul[0].header[VispMetadataKey.modulator_state] = 1
|
|
606
|
+
hdul[0].header["VSPPOLMD"] = "observe_intensity"
|
|
607
|
+
# hdul[0].header["POL_NOIS"] = 0.666
|
|
608
|
+
# hdul[0].header["POL_SENS"] = 0.666
|
|
609
|
+
|
|
610
|
+
new_tags = [
|
|
611
|
+
VispTag.task_observe(),
|
|
612
|
+
VispTag.input(),
|
|
613
|
+
VispTag.frame(),
|
|
614
|
+
VispTag.map_scan(1),
|
|
615
|
+
VispTag.raster_step(0),
|
|
616
|
+
VispTag.modstate(1),
|
|
617
|
+
VispTag.readout_exp_time(self.constants.solar_readout_exp_times[0]),
|
|
618
|
+
]
|
|
619
|
+
file_name = self.write(data=hdul, tags=new_tags, encoder=fits_hdulist_encoder)
|
|
620
|
+
final_tags = self.tags(self.scratch.workflow_base_path / file_name)
|
|
621
|
+
logger.info(f"after re-tagging tags for {str(file_name) = } are {final_tags}")
|
|
622
|
+
|
|
623
|
+
del self.constants._db_dict[VispBudName.polarimeter_mode.value]
|
|
624
|
+
self.constants._update(
|
|
625
|
+
{
|
|
626
|
+
VispBudName.num_map_scans.value: 1,
|
|
627
|
+
VispBudName.num_raster_steps.value: 1,
|
|
628
|
+
VispBudName.polarimeter_mode.value: "observe_intensity",
|
|
629
|
+
}
|
|
630
|
+
)
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
class TagModulatedSolarGainsAsScience(VispTaskBase):
|
|
634
|
+
"""Do."""
|
|
635
|
+
|
|
636
|
+
def run(self) -> None:
|
|
637
|
+
"""Do."""
|
|
638
|
+
for modstate in range(1, self.constants.num_modstates + 1):
|
|
639
|
+
tags = [
|
|
640
|
+
VispTag.task_solar_gain(),
|
|
641
|
+
VispTag.input(),
|
|
642
|
+
VispTag.frame(),
|
|
643
|
+
VispTag.modstate(modstate),
|
|
644
|
+
]
|
|
645
|
+
file_list = list(self.read(tags=tags))
|
|
646
|
+
first_hdul = fits.open(file_list[0])
|
|
647
|
+
idx = 1 if first_hdul[0].data is None else 0
|
|
648
|
+
first_header = first_hdul[idx].header
|
|
649
|
+
logger.info(f"Averaging {len(file_list)} files")
|
|
650
|
+
arrays = self.read(tags=tags, decoder=fits_array_decoder)
|
|
651
|
+
avg_array = average_numpy_arrays(arrays=arrays)
|
|
652
|
+
|
|
653
|
+
hdul = fits.HDUList([fits.PrimaryHDU(data=avg_array, header=first_header)])
|
|
654
|
+
hdul[0].header[VispMetadataKey.raster_scan_step] = 0
|
|
655
|
+
hdul[0].header[VispMetadataKey.total_raster_steps] = 1
|
|
656
|
+
hdul[0].header[VispMetadataKey.modulator_state] = modstate
|
|
657
|
+
hdul[0].header["VSPPOLMD"] = "observe_polarimetric"
|
|
658
|
+
|
|
659
|
+
new_tags = [
|
|
660
|
+
VispTag.task_observe(),
|
|
661
|
+
VispTag.input(),
|
|
662
|
+
VispTag.frame(),
|
|
663
|
+
VispTag.map_scan(1),
|
|
664
|
+
VispTag.raster_step(0),
|
|
665
|
+
VispTag.modstate(modstate),
|
|
666
|
+
VispTag.readout_exp_time(self.constants.solar_readout_exp_times[0]),
|
|
667
|
+
]
|
|
668
|
+
file_name = self.write(data=hdul, tags=new_tags, encoder=fits_hdulist_encoder)
|
|
669
|
+
final_tags = self.tags(self.scratch.workflow_base_path / file_name)
|
|
670
|
+
logger.info(f"after re-tagging tags for {str(file_name) = } are {final_tags}")
|
|
671
|
+
|
|
672
|
+
del self.constants._db_dict[VispBudName.polarimeter_mode.value]
|
|
673
|
+
self.constants._update(
|
|
674
|
+
{
|
|
675
|
+
VispBudName.num_map_scans.value: 1,
|
|
676
|
+
VispBudName.num_raster_steps.value: 1,
|
|
677
|
+
VispBudName.polarimeter_mode.value: "observe_polarimetric",
|
|
678
|
+
}
|
|
679
|
+
)
|
|
680
|
+
logger.info(f"{self.constants.correct_for_polarization = }")
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
class SaveSolarGainAsScience(SaveTaskTags):
|
|
684
|
+
@property
|
|
685
|
+
def tag_lists_to_save(self) -> list[str]:
|
|
686
|
+
return [VispTag.task_observe(), VispTag.input(), VispTag.frame()]
|
|
687
|
+
|
|
688
|
+
@property
|
|
689
|
+
def relative_save_file(self) -> str:
|
|
690
|
+
return "solar_gain_as_science.asdf"
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
def load_solar_gain_as_science_task(force_intensity_only: bool):
|
|
694
|
+
class LoadSolarGainAsScience(LoadTaskTags):
|
|
695
|
+
constants: VispConstants
|
|
696
|
+
|
|
697
|
+
@property
|
|
698
|
+
def constants_model_class(self):
|
|
699
|
+
"""Get ViSP pipeline constants."""
|
|
700
|
+
return VispConstants
|
|
701
|
+
|
|
702
|
+
@property
|
|
703
|
+
def relative_save_file(self) -> str:
|
|
704
|
+
return "solar_gain_as_science.asdf"
|
|
705
|
+
|
|
706
|
+
def run(self):
|
|
707
|
+
super().run()
|
|
708
|
+
del self.constants._db_dict[VispBudName.polarimeter_mode.value]
|
|
709
|
+
self.constants._update(
|
|
710
|
+
{
|
|
711
|
+
VispBudName.num_map_scans.value: 1,
|
|
712
|
+
VispBudName.num_raster_steps.value: 1,
|
|
713
|
+
VispBudName.polarimeter_mode.value: "observe_intensity" if force_intensity_only else "observe_polarimetric", # fmt: skip
|
|
714
|
+
}
|
|
715
|
+
)
|
|
716
|
+
logger.info(f"{self.constants.correct_for_polarization = }")
|
|
717
|
+
|
|
718
|
+
return LoadSolarGainAsScience
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
class SavePolcalAsScience(WorkflowTaskBase):
|
|
722
|
+
constants: VispConstants
|
|
723
|
+
|
|
724
|
+
@property
|
|
725
|
+
def constants_model_class(self):
|
|
726
|
+
"""Get ViSP pipeline constants."""
|
|
727
|
+
return VispConstants
|
|
728
|
+
|
|
729
|
+
@property
|
|
730
|
+
def tag_lists_to_save(self) -> list[str]:
|
|
731
|
+
return [VispTag.task_observe(), VispTag.input(), VispTag.frame()]
|
|
732
|
+
|
|
733
|
+
@property
|
|
734
|
+
def relative_save_file(self) -> str:
|
|
735
|
+
return "polcal_as_science.asdf"
|
|
736
|
+
|
|
737
|
+
def run(self):
|
|
738
|
+
file_tag_dict = dict()
|
|
739
|
+
tag_list_list = self.tag_lists_to_save
|
|
740
|
+
if isinstance(tag_list_list[0], str):
|
|
741
|
+
tag_list_list = [tag_list_list]
|
|
742
|
+
|
|
743
|
+
pcas_constants = {
|
|
744
|
+
VispBudName.num_map_scans.value: self.constants.num_map_scans,
|
|
745
|
+
VispBudName.num_raster_steps.value: self.constants.num_raster_steps,
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
for tags_to_save in tag_list_list:
|
|
749
|
+
path_list = self.read(tags=tags_to_save)
|
|
750
|
+
save_dir = self.scratch.workflow_base_path / Path(self.relative_save_file).stem
|
|
751
|
+
save_dir.mkdir(exist_ok=True)
|
|
752
|
+
for p in path_list:
|
|
753
|
+
copied_path = shutil.copy(str(p), save_dir)
|
|
754
|
+
tags = self.tags(p)
|
|
755
|
+
file_tag_dict[copied_path] = tags
|
|
756
|
+
|
|
757
|
+
full_save_file = self.scratch.workflow_base_path / self.relative_save_file
|
|
758
|
+
tree = {"file_tag_dict": file_tag_dict, "pcas_constants": pcas_constants}
|
|
759
|
+
af = asdf.AsdfFile(tree)
|
|
760
|
+
af.write_to(full_save_file)
|
|
761
|
+
logger.info(f"Saved polcal science frames to {full_save_file}")
|
|
762
|
+
|
|
763
|
+
|
|
764
|
+
class LoadPolcalAsScience(WorkflowTaskBase):
|
|
765
|
+
@property
|
|
766
|
+
def relative_save_file(self) -> str:
|
|
767
|
+
return "polcal_as_science.asdf"
|
|
768
|
+
|
|
769
|
+
def run(self):
|
|
770
|
+
full_save_file = self.scratch.workflow_base_path / self.relative_save_file
|
|
771
|
+
with asdf.open(full_save_file) as af:
|
|
772
|
+
for f, t in af.tree["file_tag_dict"].items():
|
|
773
|
+
self.tag(path=f, tags=t)
|
|
774
|
+
|
|
775
|
+
self.constants._db_dict.update(**af.tree["pcas_constants"])
|
|
776
|
+
|
|
777
|
+
logger.info(f"Loaded database entries from {full_save_file}")
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import pytest
|
|
2
2
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
3
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
4
3
|
|
|
5
4
|
from dkist_processing_visp.models.tags import VispTag
|
|
6
5
|
from dkist_processing_visp.tasks.assemble_movie import AssembleVispMovie
|
|
@@ -49,9 +48,9 @@ def assemble_task_with_tagged_movie_frames(tmp_path, recipe_run_id, init_visp_co
|
|
|
49
48
|
task._purge()
|
|
50
49
|
|
|
51
50
|
|
|
52
|
-
def test_assemble_movie(assemble_task_with_tagged_movie_frames, mocker):
|
|
51
|
+
def test_assemble_movie(assemble_task_with_tagged_movie_frames, mocker, fake_gql_client):
|
|
53
52
|
mocker.patch(
|
|
54
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
53
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
55
54
|
)
|
|
56
55
|
task, num_maps = assemble_task_with_tagged_movie_frames
|
|
57
56
|
write_movie_frames_to_task(task, num_maps)
|
|
@@ -1,27 +1,65 @@
|
|
|
1
|
+
from typing import Generator
|
|
1
2
|
from unittest.mock import MagicMock
|
|
2
3
|
|
|
4
|
+
import numpy as np
|
|
3
5
|
import pytest
|
|
6
|
+
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
7
|
+
from dkist_processing_common.codecs.asdf import asdf_encoder
|
|
8
|
+
from dkist_processing_common.codecs.quality import quality_data_decoder
|
|
4
9
|
|
|
10
|
+
from dkist_processing_visp.models.metric_code import VispMetricCode
|
|
11
|
+
from dkist_processing_visp.models.tags import VispTag
|
|
5
12
|
from dkist_processing_visp.tasks.l1_output_data import VispAssembleQualityData
|
|
6
13
|
|
|
7
14
|
|
|
8
15
|
@pytest.fixture
|
|
9
|
-
def visp_assemble_quality_data_task(
|
|
16
|
+
def visp_assemble_quality_data_task(
|
|
17
|
+
tmp_path, recipe_run_id
|
|
18
|
+
) -> Generator[VispAssembleQualityData, None, None]:
|
|
10
19
|
|
|
11
20
|
with VispAssembleQualityData(
|
|
12
21
|
recipe_run_id=recipe_run_id, workflow_name="visp_assemble_quality", workflow_version="VX.Y"
|
|
13
22
|
) as task:
|
|
23
|
+
task.scratch = WorkflowFileSystem(scratch_base_path=tmp_path, recipe_run_id=recipe_run_id)
|
|
14
24
|
yield task
|
|
15
25
|
task._purge()
|
|
16
26
|
|
|
17
27
|
|
|
28
|
+
def write_raw_vignette_metrics_to_task(task):
|
|
29
|
+
for beam in [1, 2]:
|
|
30
|
+
dummy_vec = np.arange(10)
|
|
31
|
+
first_vignette_quality_outputs = {
|
|
32
|
+
"output_wave_vec": dummy_vec,
|
|
33
|
+
"input_spectrum": dummy_vec,
|
|
34
|
+
"best_fit_atlas": dummy_vec,
|
|
35
|
+
"best_fit_continuum": dummy_vec,
|
|
36
|
+
"residuals": dummy_vec,
|
|
37
|
+
}
|
|
38
|
+
task.write(
|
|
39
|
+
data=first_vignette_quality_outputs,
|
|
40
|
+
tags=[VispTag.quality(VispMetricCode.solar_first_vignette), VispTag.beam(beam)],
|
|
41
|
+
encoder=asdf_encoder,
|
|
42
|
+
)
|
|
43
|
+
final_correction_quality_outputs = {
|
|
44
|
+
"output_wave_vec": dummy_vec,
|
|
45
|
+
"median_spec": dummy_vec,
|
|
46
|
+
"low_deviation": dummy_vec,
|
|
47
|
+
"high_deviation": dummy_vec,
|
|
48
|
+
}
|
|
49
|
+
task.write(
|
|
50
|
+
data=final_correction_quality_outputs,
|
|
51
|
+
tags=[VispTag.quality(VispMetricCode.solar_final_vignette), VispTag.beam(beam)],
|
|
52
|
+
encoder=asdf_encoder,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
18
56
|
@pytest.fixture
|
|
19
57
|
def dummy_quality_data() -> list[dict]:
|
|
20
58
|
return [{"dummy_key": "dummy_value"}]
|
|
21
59
|
|
|
22
60
|
|
|
23
61
|
@pytest.fixture
|
|
24
|
-
def
|
|
62
|
+
def common_quality_assemble_data_mock(mocker, dummy_quality_data) -> MagicMock:
|
|
25
63
|
yield mocker.patch(
|
|
26
64
|
"dkist_processing_common.tasks.mixin.quality.QualityMixin.quality_assemble_data",
|
|
27
65
|
return_value=dummy_quality_data,
|
|
@@ -29,13 +67,60 @@ def quality_assemble_data_mock(mocker, dummy_quality_data) -> MagicMock:
|
|
|
29
67
|
)
|
|
30
68
|
|
|
31
69
|
|
|
32
|
-
def
|
|
70
|
+
def test_vignette_metrics_built(visp_assemble_quality_data_task):
|
|
71
|
+
"""
|
|
72
|
+
Given: A `VispAssembleQualityData` task with raw vignette metrics in scratch
|
|
73
|
+
When: Building the quality report data
|
|
74
|
+
Then: The vignette metrics are included in the data
|
|
75
|
+
"""
|
|
76
|
+
task = visp_assemble_quality_data_task
|
|
77
|
+
write_raw_vignette_metrics_to_task(task)
|
|
78
|
+
|
|
79
|
+
task()
|
|
80
|
+
|
|
81
|
+
final_report_list = list(task.read(tags=VispTag.quality_data(), decoder=quality_data_decoder))
|
|
82
|
+
assert len(final_report_list) == 1
|
|
83
|
+
final_report = final_report_list[0]
|
|
84
|
+
|
|
85
|
+
initial_vignette_metrics = list(
|
|
86
|
+
filter(lambda i: i["name"].startswith("Initial Vignette Estimation"), final_report)
|
|
87
|
+
)
|
|
88
|
+
assert len(initial_vignette_metrics) == 2
|
|
89
|
+
facet_set = set()
|
|
90
|
+
for m in initial_vignette_metrics:
|
|
91
|
+
assert m["metric_code"] == VispMetricCode.solar_first_vignette.value
|
|
92
|
+
assert m["description"]
|
|
93
|
+
assert m["multi_plot_data"]
|
|
94
|
+
facet_set.add(m["facet"])
|
|
95
|
+
|
|
96
|
+
assert facet_set == {"BEAM_1", "BEAM_2"}
|
|
97
|
+
|
|
98
|
+
final_vignette_metrics = list(
|
|
99
|
+
filter(lambda i: i["name"].startswith("Final Vignette Estimation"), final_report)
|
|
100
|
+
)
|
|
101
|
+
assert len(final_vignette_metrics) == 2
|
|
102
|
+
facet_set = set()
|
|
103
|
+
for m in final_vignette_metrics:
|
|
104
|
+
assert m["metric_code"] == VispMetricCode.solar_final_vignette.value
|
|
105
|
+
assert m["description"]
|
|
106
|
+
assert m["multi_plot_data"]
|
|
107
|
+
facet_set.add(m["facet"])
|
|
108
|
+
|
|
109
|
+
assert facet_set == {"BEAM_1", "BEAM_2"}
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def test_correct_polcal_label_list(
|
|
113
|
+
visp_assemble_quality_data_task, common_quality_assemble_data_mock
|
|
114
|
+
):
|
|
33
115
|
"""
|
|
34
116
|
Given: A VispAssembleQualityData task
|
|
35
117
|
When: Calling the task
|
|
36
118
|
Then: The correct polcal_label_list property is passed to .quality_assemble_data
|
|
37
119
|
"""
|
|
38
120
|
task = visp_assemble_quality_data_task
|
|
121
|
+
write_raw_vignette_metrics_to_task(task)
|
|
39
122
|
|
|
40
123
|
task()
|
|
41
|
-
|
|
124
|
+
common_quality_assemble_data_mock.assert_called_once_with(
|
|
125
|
+
task, polcal_label_list=["Beam 1", "Beam 2"]
|
|
126
|
+
)
|