dkist-processing-visp 3.3.0__py3-none-any.whl → 5.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dkist_processing_visp/__init__.py +1 -0
- dkist_processing_visp/config.py +1 -0
- dkist_processing_visp/models/constants.py +52 -21
- dkist_processing_visp/models/fits_access.py +20 -0
- dkist_processing_visp/models/metric_code.py +10 -0
- dkist_processing_visp/models/parameters.py +129 -19
- dkist_processing_visp/models/tags.py +1 -0
- dkist_processing_visp/models/task_name.py +1 -0
- dkist_processing_visp/parsers/map_repeats.py +1 -0
- dkist_processing_visp/parsers/modulator_states.py +1 -0
- dkist_processing_visp/parsers/polarimeter_mode.py +3 -1
- dkist_processing_visp/parsers/raster_step.py +4 -1
- dkist_processing_visp/parsers/spectrograph_configuration.py +75 -0
- dkist_processing_visp/parsers/time.py +15 -7
- dkist_processing_visp/parsers/visp_l0_fits_access.py +19 -8
- dkist_processing_visp/parsers/visp_l1_fits_access.py +1 -0
- dkist_processing_visp/tasks/__init__.py +1 -0
- dkist_processing_visp/tasks/assemble_movie.py +1 -0
- dkist_processing_visp/tasks/background_light.py +2 -1
- dkist_processing_visp/tasks/dark.py +5 -4
- dkist_processing_visp/tasks/geometric.py +132 -20
- dkist_processing_visp/tasks/instrument_polarization.py +13 -12
- dkist_processing_visp/tasks/l1_output_data.py +203 -0
- dkist_processing_visp/tasks/lamp.py +53 -93
- dkist_processing_visp/tasks/make_movie_frames.py +8 -6
- dkist_processing_visp/tasks/mixin/beam_access.py +1 -0
- dkist_processing_visp/tasks/mixin/corrections.py +54 -4
- dkist_processing_visp/tasks/mixin/downsample.py +1 -0
- dkist_processing_visp/tasks/parse.py +34 -4
- dkist_processing_visp/tasks/quality_metrics.py +5 -4
- dkist_processing_visp/tasks/science.py +126 -46
- dkist_processing_visp/tasks/solar.py +896 -456
- dkist_processing_visp/tasks/visp_base.py +2 -0
- dkist_processing_visp/tasks/write_l1.py +25 -5
- dkist_processing_visp/tests/conftest.py +99 -35
- dkist_processing_visp/tests/header_models.py +92 -20
- dkist_processing_visp/tests/local_trial_workflows/l0_cals_only.py +4 -23
- dkist_processing_visp/tests/local_trial_workflows/l0_polcals_as_science.py +421 -0
- dkist_processing_visp/tests/local_trial_workflows/l0_solar_gain_as_science.py +10 -29
- dkist_processing_visp/tests/local_trial_workflows/l0_to_l1.py +1 -21
- dkist_processing_visp/tests/local_trial_workflows/local_trial_helpers.py +98 -14
- dkist_processing_visp/tests/test_assemble_movie.py +2 -3
- dkist_processing_visp/tests/test_assemble_quality.py +89 -4
- dkist_processing_visp/tests/test_background_light.py +8 -5
- dkist_processing_visp/tests/test_dark.py +4 -3
- dkist_processing_visp/tests/test_fits_access.py +43 -0
- dkist_processing_visp/tests/test_geometric.py +45 -4
- dkist_processing_visp/tests/test_instrument_polarization.py +4 -3
- dkist_processing_visp/tests/test_lamp.py +22 -26
- dkist_processing_visp/tests/test_make_movie_frames.py +4 -4
- dkist_processing_visp/tests/test_map_repeats.py +3 -1
- dkist_processing_visp/tests/test_parameters.py +122 -21
- dkist_processing_visp/tests/test_parse.py +98 -14
- dkist_processing_visp/tests/test_quality.py +2 -3
- dkist_processing_visp/tests/test_science.py +113 -15
- dkist_processing_visp/tests/test_solar.py +318 -99
- dkist_processing_visp/tests/test_visp_constants.py +36 -8
- dkist_processing_visp/tests/test_workflows.py +1 -0
- dkist_processing_visp/tests/test_write_l1.py +17 -3
- dkist_processing_visp/workflows/__init__.py +1 -0
- dkist_processing_visp/workflows/l0_processing.py +8 -2
- dkist_processing_visp/workflows/trial_workflows.py +8 -2
- dkist_processing_visp-5.1.1.dist-info/METADATA +552 -0
- dkist_processing_visp-5.1.1.dist-info/RECORD +94 -0
- docs/conf.py +5 -1
- docs/gain_correction.rst +50 -42
- dkist_processing_visp/tasks/mixin/line_zones.py +0 -115
- dkist_processing_visp-3.3.0.dist-info/METADATA +0 -459
- dkist_processing_visp-3.3.0.dist-info/RECORD +0 -90
- {dkist_processing_visp-3.3.0.dist-info → dkist_processing_visp-5.1.1.dist-info}/WHEEL +0 -0
- {dkist_processing_visp-3.3.0.dist-info → dkist_processing_visp-5.1.1.dist-info}/top_level.txt +0 -0
|
@@ -10,17 +10,20 @@ from dkist_header_validator import spec214_validator
|
|
|
10
10
|
from dkist_processing_common.codecs.fits import fits_array_decoder
|
|
11
11
|
from dkist_processing_common.codecs.fits import fits_hdulist_encoder
|
|
12
12
|
from dkist_processing_common.models.constants import BudName
|
|
13
|
+
from dkist_processing_common.models.fits_access import MetadataKey
|
|
13
14
|
from dkist_processing_common.models.task_name import TaskName
|
|
14
15
|
from dkist_processing_common.parsers.cs_step import CSStepFlower
|
|
15
16
|
from dkist_processing_common.parsers.cs_step import NumCSStepBud
|
|
16
17
|
from dkist_processing_common.parsers.retarder import RetarderNameBud
|
|
17
|
-
from dkist_processing_common.parsers.task import parse_header_ip_task_with_gains
|
|
18
18
|
from dkist_processing_common.parsers.task import PolcalTaskFlower
|
|
19
19
|
from dkist_processing_common.parsers.task import TaskTypeFlower
|
|
20
|
+
from dkist_processing_common.parsers.task import parse_header_ip_task_with_gains
|
|
20
21
|
from dkist_processing_common.parsers.time import ExposureTimeFlower
|
|
21
22
|
from dkist_processing_common.parsers.time import ReadoutExpTimeFlower
|
|
22
23
|
from dkist_processing_common.parsers.time import TaskExposureTimesBud
|
|
23
24
|
from dkist_processing_common.parsers.time import TaskReadoutExpTimesBud
|
|
25
|
+
from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
|
|
26
|
+
from dkist_processing_common.parsers.unique_bud import UniqueBud
|
|
24
27
|
from dkist_processing_common.tasks import ParseL0InputDataBase
|
|
25
28
|
from dkist_processing_common.tasks import WorkflowTaskBase
|
|
26
29
|
from dkist_processing_common.tasks.mixin.globus import GlobusTransferItem
|
|
@@ -30,10 +33,14 @@ from loguru import logger
|
|
|
30
33
|
|
|
31
34
|
from dkist_processing_visp.models.constants import VispBudName
|
|
32
35
|
from dkist_processing_visp.models.constants import VispConstants
|
|
36
|
+
from dkist_processing_visp.models.fits_access import VispMetadataKey
|
|
37
|
+
from dkist_processing_visp.models.metric_code import VispMetricCode
|
|
33
38
|
from dkist_processing_visp.models.parameters import VispParsingParameters
|
|
34
39
|
from dkist_processing_visp.models.tags import VispTag
|
|
35
40
|
from dkist_processing_visp.models.task_name import VispTaskName
|
|
36
41
|
from dkist_processing_visp.parsers.modulator_states import ModulatorStateFlower
|
|
42
|
+
from dkist_processing_visp.parsers.spectrograph_configuration import IncidentLightAngleBud
|
|
43
|
+
from dkist_processing_visp.parsers.spectrograph_configuration import ReflectedLightAngleBud
|
|
37
44
|
from dkist_processing_visp.parsers.time import DarkReadoutExpTimePickyBud
|
|
38
45
|
from dkist_processing_visp.parsers.time import NonDarkNonPolcalTaskReadoutExpTimesBud
|
|
39
46
|
from dkist_processing_visp.parsers.visp_l0_fits_access import VispL0FitsAccess
|
|
@@ -263,7 +270,9 @@ class SaveSolarCal(SaveTaskTags):
|
|
|
263
270
|
@property
|
|
264
271
|
def tag_lists_to_save(self) -> list[list[str]]:
|
|
265
272
|
return super().tag_lists_to_save + [
|
|
266
|
-
[VispTag.quality("TASK_TYPES"), VispTag.workflow_task("SolarCalibration")]
|
|
273
|
+
[VispTag.quality("TASK_TYPES"), VispTag.workflow_task("SolarCalibration")],
|
|
274
|
+
[VispTag.quality(VispMetricCode.solar_first_vignette)],
|
|
275
|
+
[VispTag.quality(VispMetricCode.solar_final_vignette)],
|
|
267
276
|
]
|
|
268
277
|
|
|
269
278
|
@property
|
|
@@ -415,39 +424,55 @@ class ParseCalOnlyL0InputData(ParseL0InputDataBase):
|
|
|
415
424
|
@property
|
|
416
425
|
def constant_buds(self):
|
|
417
426
|
"""Add ViSP specific constants to common constants."""
|
|
427
|
+
# TODO: Subclass ViSP parse task and *remove* unneeded things from this list
|
|
418
428
|
return super().constant_buds + [
|
|
429
|
+
UniqueBud(constant_name=VispBudName.arm_id.value, metadata_key=VispMetadataKey.arm_id),
|
|
419
430
|
NumCSStepBud(self.parameters.max_cs_step_time_sec),
|
|
420
431
|
NonDarkNonPolcalTaskReadoutExpTimesBud(),
|
|
421
432
|
DarkReadoutExpTimePickyBud(),
|
|
422
433
|
RetarderNameBud(),
|
|
434
|
+
IncidentLightAngleBud(),
|
|
435
|
+
ReflectedLightAngleBud(),
|
|
436
|
+
TaskUniqueBud(
|
|
437
|
+
constant_name=VispBudName.grating_constant_inverse_mm.value,
|
|
438
|
+
metadata_key=VispMetadataKey.grating_constant_inverse_mm,
|
|
439
|
+
ip_task_types=[TaskName.observe.value, TaskName.solar_gain.value],
|
|
440
|
+
task_type_parsing_function=parse_header_ip_task_with_gains,
|
|
441
|
+
),
|
|
442
|
+
TaskUniqueBud(
|
|
443
|
+
constant_name=VispBudName.solar_gain_ip_start_time.value,
|
|
444
|
+
metadata_key=MetadataKey.ip_start_time,
|
|
445
|
+
ip_task_types=TaskName.solar_gain,
|
|
446
|
+
task_type_parsing_function=parse_header_ip_task_with_gains,
|
|
447
|
+
),
|
|
423
448
|
TaskExposureTimesBud(
|
|
424
449
|
stem_name=VispBudName.lamp_exposure_times.value,
|
|
425
|
-
|
|
450
|
+
ip_task_types=TaskName.lamp_gain.value,
|
|
426
451
|
header_task_parsing_func=parse_header_ip_task_with_gains,
|
|
427
452
|
),
|
|
428
453
|
TaskExposureTimesBud(
|
|
429
454
|
stem_name=VispBudName.solar_exposure_times.value,
|
|
430
|
-
|
|
455
|
+
ip_task_types=TaskName.solar_gain.value,
|
|
431
456
|
header_task_parsing_func=parse_header_ip_task_with_gains,
|
|
432
457
|
),
|
|
433
458
|
TaskExposureTimesBud(
|
|
434
459
|
stem_name=VispBudName.polcal_exposure_times.value,
|
|
435
|
-
|
|
460
|
+
ip_task_types=TaskName.polcal.value,
|
|
436
461
|
header_task_parsing_func=parse_header_ip_task_with_gains,
|
|
437
462
|
),
|
|
438
463
|
TaskReadoutExpTimesBud(
|
|
439
464
|
stem_name=VispBudName.lamp_readout_exp_times.value,
|
|
440
|
-
|
|
465
|
+
ip_task_types=TaskName.lamp_gain.value,
|
|
441
466
|
header_task_parsing_func=parse_header_ip_task_with_gains,
|
|
442
467
|
),
|
|
443
468
|
TaskReadoutExpTimesBud(
|
|
444
469
|
stem_name=VispBudName.solar_readout_exp_times.value,
|
|
445
|
-
|
|
470
|
+
ip_task_types=TaskName.solar_gain.value,
|
|
446
471
|
header_task_parsing_func=parse_header_ip_task_with_gains,
|
|
447
472
|
),
|
|
448
473
|
TaskReadoutExpTimesBud(
|
|
449
474
|
stem_name=VispBudName.polcal_readout_exp_times.value,
|
|
450
|
-
|
|
475
|
+
ip_task_types=TaskName.polcal.value,
|
|
451
476
|
header_task_parsing_func=parse_header_ip_task_with_gains,
|
|
452
477
|
),
|
|
453
478
|
]
|
|
@@ -575,9 +600,9 @@ class TagSingleSolarGainAsScience(VispTaskBase):
|
|
|
575
600
|
avg_array = average_numpy_arrays(arrays=arrays)
|
|
576
601
|
|
|
577
602
|
hdul = fits.HDUList([fits.PrimaryHDU(data=avg_array, header=first_header)])
|
|
578
|
-
hdul[0].header[
|
|
579
|
-
hdul[0].header[
|
|
580
|
-
hdul[0].header[
|
|
603
|
+
hdul[0].header[VispMetadataKey.raster_scan_step] = 0
|
|
604
|
+
hdul[0].header[VispMetadataKey.total_raster_steps] = 1
|
|
605
|
+
hdul[0].header[VispMetadataKey.modulator_state] = 1
|
|
581
606
|
hdul[0].header["VSPPOLMD"] = "observe_intensity"
|
|
582
607
|
# hdul[0].header["POL_NOIS"] = 0.666
|
|
583
608
|
# hdul[0].header["POL_SENS"] = 0.666
|
|
@@ -626,9 +651,9 @@ class TagModulatedSolarGainsAsScience(VispTaskBase):
|
|
|
626
651
|
avg_array = average_numpy_arrays(arrays=arrays)
|
|
627
652
|
|
|
628
653
|
hdul = fits.HDUList([fits.PrimaryHDU(data=avg_array, header=first_header)])
|
|
629
|
-
hdul[0].header[
|
|
630
|
-
hdul[0].header[
|
|
631
|
-
hdul[0].header[
|
|
654
|
+
hdul[0].header[VispMetadataKey.raster_scan_step] = 0
|
|
655
|
+
hdul[0].header[VispMetadataKey.total_raster_steps] = 1
|
|
656
|
+
hdul[0].header[VispMetadataKey.modulator_state] = modstate
|
|
632
657
|
hdul[0].header["VSPPOLMD"] = "observe_polarimetric"
|
|
633
658
|
|
|
634
659
|
new_tags = [
|
|
@@ -691,3 +716,62 @@ def load_solar_gain_as_science_task(force_intensity_only: bool):
|
|
|
691
716
|
logger.info(f"{self.constants.correct_for_polarization = }")
|
|
692
717
|
|
|
693
718
|
return LoadSolarGainAsScience
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
class SavePolcalAsScience(WorkflowTaskBase):
|
|
722
|
+
constants: VispConstants
|
|
723
|
+
|
|
724
|
+
@property
|
|
725
|
+
def constants_model_class(self):
|
|
726
|
+
"""Get ViSP pipeline constants."""
|
|
727
|
+
return VispConstants
|
|
728
|
+
|
|
729
|
+
@property
|
|
730
|
+
def tag_lists_to_save(self) -> list[str]:
|
|
731
|
+
return [VispTag.task_observe(), VispTag.input(), VispTag.frame()]
|
|
732
|
+
|
|
733
|
+
@property
|
|
734
|
+
def relative_save_file(self) -> str:
|
|
735
|
+
return "polcal_as_science.asdf"
|
|
736
|
+
|
|
737
|
+
def run(self):
|
|
738
|
+
file_tag_dict = dict()
|
|
739
|
+
tag_list_list = self.tag_lists_to_save
|
|
740
|
+
if isinstance(tag_list_list[0], str):
|
|
741
|
+
tag_list_list = [tag_list_list]
|
|
742
|
+
|
|
743
|
+
pcas_constants = {
|
|
744
|
+
VispBudName.num_map_scans.value: self.constants.num_map_scans,
|
|
745
|
+
VispBudName.num_raster_steps.value: self.constants.num_raster_steps,
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
for tags_to_save in tag_list_list:
|
|
749
|
+
path_list = self.read(tags=tags_to_save)
|
|
750
|
+
save_dir = self.scratch.workflow_base_path / Path(self.relative_save_file).stem
|
|
751
|
+
save_dir.mkdir(exist_ok=True)
|
|
752
|
+
for p in path_list:
|
|
753
|
+
copied_path = shutil.copy(str(p), save_dir)
|
|
754
|
+
tags = self.tags(p)
|
|
755
|
+
file_tag_dict[copied_path] = tags
|
|
756
|
+
|
|
757
|
+
full_save_file = self.scratch.workflow_base_path / self.relative_save_file
|
|
758
|
+
tree = {"file_tag_dict": file_tag_dict, "pcas_constants": pcas_constants}
|
|
759
|
+
af = asdf.AsdfFile(tree)
|
|
760
|
+
af.write_to(full_save_file)
|
|
761
|
+
logger.info(f"Saved polcal science frames to {full_save_file}")
|
|
762
|
+
|
|
763
|
+
|
|
764
|
+
class LoadPolcalAsScience(WorkflowTaskBase):
|
|
765
|
+
@property
|
|
766
|
+
def relative_save_file(self) -> str:
|
|
767
|
+
return "polcal_as_science.asdf"
|
|
768
|
+
|
|
769
|
+
def run(self):
|
|
770
|
+
full_save_file = self.scratch.workflow_base_path / self.relative_save_file
|
|
771
|
+
with asdf.open(full_save_file) as af:
|
|
772
|
+
for f, t in af.tree["file_tag_dict"].items():
|
|
773
|
+
self.tag(path=f, tags=t)
|
|
774
|
+
|
|
775
|
+
self.constants._db_dict.update(**af.tree["pcas_constants"])
|
|
776
|
+
|
|
777
|
+
logger.info(f"Loaded database entries from {full_save_file}")
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import pytest
|
|
2
2
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
3
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
4
3
|
|
|
5
4
|
from dkist_processing_visp.models.tags import VispTag
|
|
6
5
|
from dkist_processing_visp.tasks.assemble_movie import AssembleVispMovie
|
|
@@ -49,9 +48,9 @@ def assemble_task_with_tagged_movie_frames(tmp_path, recipe_run_id, init_visp_co
|
|
|
49
48
|
task._purge()
|
|
50
49
|
|
|
51
50
|
|
|
52
|
-
def test_assemble_movie(assemble_task_with_tagged_movie_frames, mocker):
|
|
51
|
+
def test_assemble_movie(assemble_task_with_tagged_movie_frames, mocker, fake_gql_client):
|
|
53
52
|
mocker.patch(
|
|
54
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
53
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
55
54
|
)
|
|
56
55
|
task, num_maps = assemble_task_with_tagged_movie_frames
|
|
57
56
|
write_movie_frames_to_task(task, num_maps)
|
|
@@ -1,27 +1,65 @@
|
|
|
1
|
+
from typing import Generator
|
|
1
2
|
from unittest.mock import MagicMock
|
|
2
3
|
|
|
4
|
+
import numpy as np
|
|
3
5
|
import pytest
|
|
6
|
+
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
7
|
+
from dkist_processing_common.codecs.asdf import asdf_encoder
|
|
8
|
+
from dkist_processing_common.codecs.quality import quality_data_decoder
|
|
4
9
|
|
|
10
|
+
from dkist_processing_visp.models.metric_code import VispMetricCode
|
|
11
|
+
from dkist_processing_visp.models.tags import VispTag
|
|
5
12
|
from dkist_processing_visp.tasks.l1_output_data import VispAssembleQualityData
|
|
6
13
|
|
|
7
14
|
|
|
8
15
|
@pytest.fixture
|
|
9
|
-
def visp_assemble_quality_data_task(
|
|
16
|
+
def visp_assemble_quality_data_task(
|
|
17
|
+
tmp_path, recipe_run_id
|
|
18
|
+
) -> Generator[VispAssembleQualityData, None, None]:
|
|
10
19
|
|
|
11
20
|
with VispAssembleQualityData(
|
|
12
21
|
recipe_run_id=recipe_run_id, workflow_name="visp_assemble_quality", workflow_version="VX.Y"
|
|
13
22
|
) as task:
|
|
23
|
+
task.scratch = WorkflowFileSystem(scratch_base_path=tmp_path, recipe_run_id=recipe_run_id)
|
|
14
24
|
yield task
|
|
15
25
|
task._purge()
|
|
16
26
|
|
|
17
27
|
|
|
28
|
+
def write_raw_vignette_metrics_to_task(task):
|
|
29
|
+
for beam in [1, 2]:
|
|
30
|
+
dummy_vec = np.arange(10)
|
|
31
|
+
first_vignette_quality_outputs = {
|
|
32
|
+
"output_wave_vec": dummy_vec,
|
|
33
|
+
"input_spectrum": dummy_vec,
|
|
34
|
+
"best_fit_atlas": dummy_vec,
|
|
35
|
+
"best_fit_continuum": dummy_vec,
|
|
36
|
+
"residuals": dummy_vec,
|
|
37
|
+
}
|
|
38
|
+
task.write(
|
|
39
|
+
data=first_vignette_quality_outputs,
|
|
40
|
+
tags=[VispTag.quality(VispMetricCode.solar_first_vignette), VispTag.beam(beam)],
|
|
41
|
+
encoder=asdf_encoder,
|
|
42
|
+
)
|
|
43
|
+
final_correction_quality_outputs = {
|
|
44
|
+
"output_wave_vec": dummy_vec,
|
|
45
|
+
"median_spec": dummy_vec,
|
|
46
|
+
"low_deviation": dummy_vec,
|
|
47
|
+
"high_deviation": dummy_vec,
|
|
48
|
+
}
|
|
49
|
+
task.write(
|
|
50
|
+
data=final_correction_quality_outputs,
|
|
51
|
+
tags=[VispTag.quality(VispMetricCode.solar_final_vignette), VispTag.beam(beam)],
|
|
52
|
+
encoder=asdf_encoder,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
18
56
|
@pytest.fixture
|
|
19
57
|
def dummy_quality_data() -> list[dict]:
|
|
20
58
|
return [{"dummy_key": "dummy_value"}]
|
|
21
59
|
|
|
22
60
|
|
|
23
61
|
@pytest.fixture
|
|
24
|
-
def
|
|
62
|
+
def common_quality_assemble_data_mock(mocker, dummy_quality_data) -> MagicMock:
|
|
25
63
|
yield mocker.patch(
|
|
26
64
|
"dkist_processing_common.tasks.mixin.quality.QualityMixin.quality_assemble_data",
|
|
27
65
|
return_value=dummy_quality_data,
|
|
@@ -29,13 +67,60 @@ def quality_assemble_data_mock(mocker, dummy_quality_data) -> MagicMock:
|
|
|
29
67
|
)
|
|
30
68
|
|
|
31
69
|
|
|
32
|
-
def
|
|
70
|
+
def test_vignette_metrics_built(visp_assemble_quality_data_task):
|
|
71
|
+
"""
|
|
72
|
+
Given: A `VispAssembleQualityData` task with raw vignette metrics in scratch
|
|
73
|
+
When: Building the quality report data
|
|
74
|
+
Then: The vignette metrics are included in the data
|
|
75
|
+
"""
|
|
76
|
+
task = visp_assemble_quality_data_task
|
|
77
|
+
write_raw_vignette_metrics_to_task(task)
|
|
78
|
+
|
|
79
|
+
task()
|
|
80
|
+
|
|
81
|
+
final_report_list = list(task.read(tags=VispTag.quality_data(), decoder=quality_data_decoder))
|
|
82
|
+
assert len(final_report_list) == 1
|
|
83
|
+
final_report = final_report_list[0]
|
|
84
|
+
|
|
85
|
+
initial_vignette_metrics = list(
|
|
86
|
+
filter(lambda i: i["name"].startswith("Initial Vignette Estimation"), final_report)
|
|
87
|
+
)
|
|
88
|
+
assert len(initial_vignette_metrics) == 2
|
|
89
|
+
facet_set = set()
|
|
90
|
+
for m in initial_vignette_metrics:
|
|
91
|
+
assert m["metric_code"] == VispMetricCode.solar_first_vignette.value
|
|
92
|
+
assert m["description"]
|
|
93
|
+
assert m["multi_plot_data"]
|
|
94
|
+
facet_set.add(m["facet"])
|
|
95
|
+
|
|
96
|
+
assert facet_set == {"BEAM_1", "BEAM_2"}
|
|
97
|
+
|
|
98
|
+
final_vignette_metrics = list(
|
|
99
|
+
filter(lambda i: i["name"].startswith("Final Vignette Estimation"), final_report)
|
|
100
|
+
)
|
|
101
|
+
assert len(final_vignette_metrics) == 2
|
|
102
|
+
facet_set = set()
|
|
103
|
+
for m in final_vignette_metrics:
|
|
104
|
+
assert m["metric_code"] == VispMetricCode.solar_final_vignette.value
|
|
105
|
+
assert m["description"]
|
|
106
|
+
assert m["multi_plot_data"]
|
|
107
|
+
facet_set.add(m["facet"])
|
|
108
|
+
|
|
109
|
+
assert facet_set == {"BEAM_1", "BEAM_2"}
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def test_correct_polcal_label_list(
|
|
113
|
+
visp_assemble_quality_data_task, common_quality_assemble_data_mock
|
|
114
|
+
):
|
|
33
115
|
"""
|
|
34
116
|
Given: A VispAssembleQualityData task
|
|
35
117
|
When: Calling the task
|
|
36
118
|
Then: The correct polcal_label_list property is passed to .quality_assemble_data
|
|
37
119
|
"""
|
|
38
120
|
task = visp_assemble_quality_data_task
|
|
121
|
+
write_raw_vignette_metrics_to_task(task)
|
|
39
122
|
|
|
40
123
|
task()
|
|
41
|
-
|
|
124
|
+
common_quality_assemble_data_mock.assert_called_once_with(
|
|
125
|
+
task, polcal_label_list=["Beam 1", "Beam 2"]
|
|
126
|
+
)
|
|
@@ -12,7 +12,6 @@ from dkist_header_validator.translator import translate_spec122_to_spec214_l0
|
|
|
12
12
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
13
13
|
from dkist_processing_common.codecs.fits import fits_array_encoder
|
|
14
14
|
from dkist_processing_common.models.task_name import TaskName
|
|
15
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
16
15
|
from dkist_processing_pac.fitter.fitter_parameters import PolcalDresserParameters
|
|
17
16
|
from dkist_processing_pac.input_data.drawer import Drawer
|
|
18
17
|
from dkist_processing_pac.input_data.dresser import Dresser
|
|
@@ -401,6 +400,7 @@ def test_background_light_calibration_task(
|
|
|
401
400
|
num_polcal_dark_files,
|
|
402
401
|
background_light,
|
|
403
402
|
mocker,
|
|
403
|
+
fake_gql_client,
|
|
404
404
|
):
|
|
405
405
|
"""
|
|
406
406
|
Give: a BackgroundLightCalibrationTask with a valid set of polcal data
|
|
@@ -408,7 +408,7 @@ def test_background_light_calibration_task(
|
|
|
408
408
|
Then: BACKGROUND intermediate frames are generated for each beam
|
|
409
409
|
"""
|
|
410
410
|
mocker.patch(
|
|
411
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
411
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
412
412
|
)
|
|
413
413
|
task_generator = background_light_calibration_task_factory()
|
|
414
414
|
task = next(task_generator)
|
|
@@ -442,6 +442,7 @@ def test_background_light_bad_exposure_times(
|
|
|
442
442
|
background_light_calibration_task_factory,
|
|
443
443
|
constants_bad_exp_times,
|
|
444
444
|
mocker,
|
|
445
|
+
fake_gql_client,
|
|
445
446
|
):
|
|
446
447
|
"""
|
|
447
448
|
Given: A set of data where the polcal exposure times don't match those of solar gain and observe
|
|
@@ -449,7 +450,7 @@ def test_background_light_bad_exposure_times(
|
|
|
449
450
|
Then: An error is raised
|
|
450
451
|
"""
|
|
451
452
|
mocker.patch(
|
|
452
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
453
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
453
454
|
)
|
|
454
455
|
task_generator = background_light_calibration_task_factory(constants=constants_bad_exp_times)
|
|
455
456
|
task = next(task_generator)
|
|
@@ -461,6 +462,7 @@ def test_background_light_non_polarimetric_dataset(
|
|
|
461
462
|
background_light_calibration_task_factory,
|
|
462
463
|
constants_nor_polarimetric,
|
|
463
464
|
mocker,
|
|
465
|
+
fake_gql_client,
|
|
464
466
|
):
|
|
465
467
|
"""
|
|
466
468
|
Given: A dataset that is non-polarimetric (i.e., Stokes-I only)
|
|
@@ -468,7 +470,7 @@ def test_background_light_non_polarimetric_dataset(
|
|
|
468
470
|
Then: Nothing is done and no files are written
|
|
469
471
|
"""
|
|
470
472
|
mocker.patch(
|
|
471
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
473
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
472
474
|
)
|
|
473
475
|
|
|
474
476
|
task_generator = background_light_calibration_task_factory(constants=constants_nor_polarimetric)
|
|
@@ -485,6 +487,7 @@ def test_background_light_non_polarimetric_dataset(
|
|
|
485
487
|
def test_background_light_switch_off(
|
|
486
488
|
background_light_calibration_task_factory,
|
|
487
489
|
mocker,
|
|
490
|
+
fake_gql_client,
|
|
488
491
|
):
|
|
489
492
|
"""
|
|
490
493
|
Given: A task with the background light switch turned off
|
|
@@ -492,7 +495,7 @@ def test_background_light_switch_off(
|
|
|
492
495
|
Then: Nothing is done and no files are written
|
|
493
496
|
"""
|
|
494
497
|
mocker.patch(
|
|
495
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
498
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
496
499
|
)
|
|
497
500
|
task_generator = background_light_calibration_task_factory(background_on=False)
|
|
498
501
|
task = next(task_generator)
|
|
@@ -6,7 +6,6 @@ import pytest
|
|
|
6
6
|
from astropy.io import fits
|
|
7
7
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
8
8
|
from dkist_processing_common.models.tags import Tag
|
|
9
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
10
9
|
|
|
11
10
|
from dkist_processing_visp.models.tags import VispTag
|
|
12
11
|
from dkist_processing_visp.tasks.dark import DarkCalibration
|
|
@@ -90,14 +89,16 @@ def dark_calibration_task(tmp_path, init_visp_constants_db, recipe_run_id):
|
|
|
90
89
|
task._purge()
|
|
91
90
|
|
|
92
91
|
|
|
93
|
-
def test_dark_calibration_task(
|
|
92
|
+
def test_dark_calibration_task(
|
|
93
|
+
dark_calibration_task, assign_input_dataset_doc_to_task, mocker, fake_gql_client
|
|
94
|
+
):
|
|
94
95
|
"""
|
|
95
96
|
Given: A DarkCalibration task with multiple task exposure times
|
|
96
97
|
When: Calling the task instance
|
|
97
98
|
Then: Only one average intermediate dark frame exists for each exposure time and unused times are not made
|
|
98
99
|
"""
|
|
99
100
|
mocker.patch(
|
|
100
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
101
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
101
102
|
)
|
|
102
103
|
# Given
|
|
103
104
|
task, readout_exp_times = dark_calibration_task
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from dkist_header_validator.translator import translate_spec122_to_spec214_l0
|
|
3
|
+
|
|
4
|
+
from dkist_processing_visp.models.fits_access import VispMetadataKey
|
|
5
|
+
from dkist_processing_visp.parsers.visp_l0_fits_access import VispL0FitsAccess
|
|
6
|
+
from dkist_processing_visp.parsers.visp_l1_fits_access import VispL1FitsAccess
|
|
7
|
+
from dkist_processing_visp.tests.header_models import VispHeadersValidObserveFrames
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@pytest.fixture(scope="session")
|
|
11
|
+
def complete_header():
|
|
12
|
+
dataset = VispHeadersValidObserveFrames(
|
|
13
|
+
array_shape=(1, 1, 1),
|
|
14
|
+
time_delta=10,
|
|
15
|
+
num_maps=1,
|
|
16
|
+
num_raster_steps=1,
|
|
17
|
+
num_modstates=1,
|
|
18
|
+
)
|
|
19
|
+
header = translate_spec122_to_spec214_l0(dataset.header())
|
|
20
|
+
return header
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_metadata_keys_in_access_bases(complete_header):
|
|
24
|
+
"""
|
|
25
|
+
Given: the set of metadata key names in VispMetadataKey
|
|
26
|
+
When: the ViSP FITS access classes define a set of new attributes
|
|
27
|
+
Then: the sets are the same and the attributes have the correct values
|
|
28
|
+
"""
|
|
29
|
+
# Given
|
|
30
|
+
visp_metadata_key_names = {vmk.name for vmk in VispMetadataKey}
|
|
31
|
+
# When
|
|
32
|
+
all_visp_fits_access_attrs = set()
|
|
33
|
+
for access_class in [VispL0FitsAccess, VispL1FitsAccess]:
|
|
34
|
+
fits_obj = access_class.from_header(complete_header)
|
|
35
|
+
visp_instance_attrs = set(vars(fits_obj).keys())
|
|
36
|
+
parent_class = access_class.mro()[1]
|
|
37
|
+
parent_fits_obj = parent_class.from_header(complete_header)
|
|
38
|
+
parent_instance_attrs = set(vars(parent_fits_obj).keys())
|
|
39
|
+
visp_fits_access_attrs = visp_instance_attrs - parent_instance_attrs
|
|
40
|
+
for attr in visp_fits_access_attrs:
|
|
41
|
+
assert getattr(fits_obj, attr) == fits_obj.header[VispMetadataKey[attr]]
|
|
42
|
+
all_visp_fits_access_attrs |= visp_fits_access_attrs
|
|
43
|
+
assert visp_metadata_key_names == all_visp_fits_access_attrs
|
|
@@ -7,14 +7,13 @@ import pytest
|
|
|
7
7
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
8
8
|
from dkist_processing_common.codecs.fits import fits_array_decoder
|
|
9
9
|
from dkist_processing_common.models.tags import Tag
|
|
10
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
11
10
|
from dkist_processing_math import transform
|
|
12
11
|
|
|
13
12
|
from dkist_processing_visp.models.tags import VispTag
|
|
14
13
|
from dkist_processing_visp.tasks.geometric import GeometricCalibration
|
|
15
|
-
from dkist_processing_visp.tests.conftest import tag_on_modstate
|
|
16
14
|
from dkist_processing_visp.tests.conftest import VispConstantsDb
|
|
17
15
|
from dkist_processing_visp.tests.conftest import VispInputDatasetParameterValues
|
|
16
|
+
from dkist_processing_visp.tests.conftest import tag_on_modstate
|
|
18
17
|
from dkist_processing_visp.tests.conftest import write_frames_to_task
|
|
19
18
|
from dkist_processing_visp.tests.conftest import write_intermediate_darks_to_task
|
|
20
19
|
from dkist_processing_visp.tests.header_models import VispHeadersInputLampGainFrames
|
|
@@ -189,7 +188,9 @@ def geometric_calibration_task(tmp_path, recipe_run_id, init_visp_constants_db):
|
|
|
189
188
|
task._purge()
|
|
190
189
|
|
|
191
190
|
|
|
192
|
-
def test_geometric_task(
|
|
191
|
+
def test_geometric_task(
|
|
192
|
+
geometric_calibration_task, assign_input_dataset_doc_to_task, mocker, fake_gql_client
|
|
193
|
+
):
|
|
193
194
|
"""
|
|
194
195
|
Given: A set of raw solar gain images and necessary intermediate calibrations
|
|
195
196
|
When: Running the geometric task
|
|
@@ -201,7 +202,7 @@ def test_geometric_task(geometric_calibration_task, assign_input_dataset_doc_to_
|
|
|
201
202
|
# tests for that. In other words, this fixture just tests if the machinery of the task completes and some object
|
|
202
203
|
# (ANY object) is written correctly.
|
|
203
204
|
mocker.patch(
|
|
204
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
205
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
205
206
|
)
|
|
206
207
|
task, readout_exp_times, num_modstates = geometric_calibration_task
|
|
207
208
|
dark_signal = 3.0
|
|
@@ -357,3 +358,43 @@ def test_basic_corrections(geometric_calibration_task, assign_input_dataset_doc_
|
|
|
357
358
|
lamp_array = task.basic_corrected_lamp_data(beam=beam, modstate=modstate)
|
|
358
359
|
np.testing.assert_equal(expected, solar_array)
|
|
359
360
|
np.testing.assert_equal(expected, lamp_array)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def test_line_zones(geometric_calibration_task):
|
|
364
|
+
"""
|
|
365
|
+
Given: A spectrum with some absorption lines
|
|
366
|
+
When: Computing zones around the lines
|
|
367
|
+
Then: Correct results are returned
|
|
368
|
+
"""
|
|
369
|
+
|
|
370
|
+
# NOTE that it does not test for removal of overlapping regions
|
|
371
|
+
def gaussian(x, amp, mu, sig):
|
|
372
|
+
return amp * np.exp(-np.power(x - mu, 2.0) / (2 * np.power(sig, 2.0)))
|
|
373
|
+
|
|
374
|
+
spec = np.ones(1000) * 100
|
|
375
|
+
x = np.arange(1000.0)
|
|
376
|
+
expected = []
|
|
377
|
+
for m, s in zip([100.0, 300.0, 700], [10.0, 20.0, 5.0]):
|
|
378
|
+
spec -= gaussian(x, 40, m, s)
|
|
379
|
+
hwhm = s * 2.355 / 2
|
|
380
|
+
expected.append((np.floor(m - hwhm).astype(int), np.ceil(m + hwhm).astype(int)))
|
|
381
|
+
|
|
382
|
+
task = geometric_calibration_task[0]
|
|
383
|
+
|
|
384
|
+
zones = task.compute_line_zones(spec[:, None], bg_order=0, rel_height=0.5)
|
|
385
|
+
assert zones == expected
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def test_identify_overlapping_zones(geometric_calibration_task):
|
|
389
|
+
"""
|
|
390
|
+
Given: A list of zone borders that contain overlapping zones
|
|
391
|
+
When: Identifying zones that overlap
|
|
392
|
+
Then: The smaller of the overlapping zones are identified for removal
|
|
393
|
+
"""
|
|
394
|
+
rips = np.array([100, 110, 220, 200])
|
|
395
|
+
lips = np.array([150, 120, 230, 250])
|
|
396
|
+
|
|
397
|
+
task = geometric_calibration_task[0]
|
|
398
|
+
|
|
399
|
+
idx_to_remove = task.identify_overlapping_zones(rips, lips)
|
|
400
|
+
assert idx_to_remove == [1, 2]
|
|
@@ -6,7 +6,6 @@ import pytest
|
|
|
6
6
|
from astropy.io import fits
|
|
7
7
|
from dkist_data_simulator.spec122 import Spec122Dataset
|
|
8
8
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
9
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
10
9
|
from dkist_processing_pac.fitter.polcal_fitter import PolcalFitter
|
|
11
10
|
from dkist_processing_pac.input_data.dresser import Dresser
|
|
12
11
|
|
|
@@ -213,6 +212,7 @@ def test_instrument_polarization_calibration_task(
|
|
|
213
212
|
background_on,
|
|
214
213
|
assign_input_dataset_doc_to_task,
|
|
215
214
|
mocker,
|
|
215
|
+
fake_gql_client,
|
|
216
216
|
):
|
|
217
217
|
"""
|
|
218
218
|
Given: An InstrumentPolarizationCalibration task
|
|
@@ -221,7 +221,7 @@ def test_instrument_polarization_calibration_task(
|
|
|
221
221
|
"""
|
|
222
222
|
|
|
223
223
|
mocker.patch(
|
|
224
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
224
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
225
225
|
)
|
|
226
226
|
mocker.patch(
|
|
227
227
|
"dkist_processing_visp.tasks.instrument_polarization.PolcalFitter",
|
|
@@ -255,7 +255,8 @@ def test_instrument_polarization_calibration_task(
|
|
|
255
255
|
task=task, num_modstates=num_modstates, data_shape=intermediate_shape
|
|
256
256
|
)
|
|
257
257
|
write_dummy_intermediate_solar_cals_to_task(
|
|
258
|
-
task=task,
|
|
258
|
+
task=task,
|
|
259
|
+
data_shape=intermediate_shape,
|
|
259
260
|
)
|
|
260
261
|
write_input_polcals_to_task(
|
|
261
262
|
task=task,
|