dkist-processing-visp 2.20.14__py3-none-any.whl → 5.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dkist_processing_visp/__init__.py +1 -0
- dkist_processing_visp/config.py +1 -0
- dkist_processing_visp/models/constants.py +61 -20
- dkist_processing_visp/models/fits_access.py +20 -0
- dkist_processing_visp/models/metric_code.py +10 -0
- dkist_processing_visp/models/parameters.py +129 -24
- dkist_processing_visp/models/tags.py +22 -1
- dkist_processing_visp/models/task_name.py +1 -0
- dkist_processing_visp/parsers/map_repeats.py +1 -0
- dkist_processing_visp/parsers/modulator_states.py +1 -0
- dkist_processing_visp/parsers/polarimeter_mode.py +4 -2
- dkist_processing_visp/parsers/raster_step.py +4 -1
- dkist_processing_visp/parsers/spectrograph_configuration.py +75 -0
- dkist_processing_visp/parsers/time.py +24 -14
- dkist_processing_visp/parsers/visp_l0_fits_access.py +19 -8
- dkist_processing_visp/parsers/visp_l1_fits_access.py +1 -0
- dkist_processing_visp/tasks/__init__.py +1 -0
- dkist_processing_visp/tasks/assemble_movie.py +1 -0
- dkist_processing_visp/tasks/background_light.py +2 -1
- dkist_processing_visp/tasks/dark.py +5 -4
- dkist_processing_visp/tasks/geometric.py +132 -20
- dkist_processing_visp/tasks/instrument_polarization.py +128 -18
- dkist_processing_visp/tasks/l1_output_data.py +203 -0
- dkist_processing_visp/tasks/lamp.py +53 -93
- dkist_processing_visp/tasks/make_movie_frames.py +8 -6
- dkist_processing_visp/tasks/mixin/beam_access.py +1 -0
- dkist_processing_visp/tasks/mixin/corrections.py +54 -4
- dkist_processing_visp/tasks/mixin/downsample.py +1 -0
- dkist_processing_visp/tasks/parse.py +50 -17
- dkist_processing_visp/tasks/quality_metrics.py +5 -4
- dkist_processing_visp/tasks/science.py +126 -46
- dkist_processing_visp/tasks/solar.py +896 -456
- dkist_processing_visp/tasks/visp_base.py +4 -3
- dkist_processing_visp/tasks/write_l1.py +38 -10
- dkist_processing_visp/tests/conftest.py +145 -47
- dkist_processing_visp/tests/header_models.py +157 -20
- dkist_processing_visp/tests/local_trial_workflows/l0_cals_only.py +21 -78
- dkist_processing_visp/tests/local_trial_workflows/l0_polcals_as_science.py +421 -0
- dkist_processing_visp/tests/local_trial_workflows/l0_solar_gain_as_science.py +387 -0
- dkist_processing_visp/tests/local_trial_workflows/l0_to_l1.py +18 -75
- dkist_processing_visp/tests/local_trial_workflows/local_trial_helpers.py +346 -14
- dkist_processing_visp/tests/test_assemble_movie.py +2 -3
- dkist_processing_visp/tests/test_assemble_quality.py +89 -4
- dkist_processing_visp/tests/test_background_light.py +51 -44
- dkist_processing_visp/tests/test_dark.py +4 -3
- dkist_processing_visp/tests/test_downsample.py +1 -0
- dkist_processing_visp/tests/test_fits_access.py +43 -0
- dkist_processing_visp/tests/test_geometric.py +45 -4
- dkist_processing_visp/tests/test_instrument_polarization.py +72 -9
- dkist_processing_visp/tests/test_lamp.py +22 -26
- dkist_processing_visp/tests/test_make_movie_frames.py +4 -4
- dkist_processing_visp/tests/test_map_repeats.py +3 -1
- dkist_processing_visp/tests/test_parameters.py +122 -21
- dkist_processing_visp/tests/test_parse.py +164 -18
- dkist_processing_visp/tests/test_quality.py +3 -4
- dkist_processing_visp/tests/test_science.py +113 -15
- dkist_processing_visp/tests/test_solar.py +318 -99
- dkist_processing_visp/tests/test_visp_constants.py +38 -8
- dkist_processing_visp/tests/test_workflows.py +1 -0
- dkist_processing_visp/tests/test_write_l1.py +22 -3
- dkist_processing_visp/workflows/__init__.py +1 -0
- dkist_processing_visp/workflows/l0_processing.py +10 -3
- dkist_processing_visp/workflows/trial_workflows.py +8 -2
- dkist_processing_visp-5.1.1.dist-info/METADATA +552 -0
- dkist_processing_visp-5.1.1.dist-info/RECORD +94 -0
- {dkist_processing_visp-2.20.14.dist-info → dkist_processing_visp-5.1.1.dist-info}/WHEEL +1 -1
- docs/conf.py +5 -1
- docs/gain_correction.rst +52 -44
- docs/science_calibration.rst +7 -0
- dkist_processing_visp/tasks/mixin/line_zones.py +0 -115
- dkist_processing_visp-2.20.14.dist-info/METADATA +0 -196
- dkist_processing_visp-2.20.14.dist-info/RECORD +0 -89
- {dkist_processing_visp-2.20.14.dist-info → dkist_processing_visp-5.1.1.dist-info}/top_level.txt +0 -0
|
@@ -12,7 +12,6 @@ from dkist_header_validator.translator import translate_spec122_to_spec214_l0
|
|
|
12
12
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
13
13
|
from dkist_processing_common.codecs.fits import fits_array_encoder
|
|
14
14
|
from dkist_processing_common.models.task_name import TaskName
|
|
15
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
16
15
|
from dkist_processing_pac.fitter.fitter_parameters import PolcalDresserParameters
|
|
17
16
|
from dkist_processing_pac.input_data.drawer import Drawer
|
|
18
17
|
from dkist_processing_pac.input_data.dresser import Dresser
|
|
@@ -351,7 +350,7 @@ def constants_bad_exp_times() -> VispConstantsDb:
|
|
|
351
350
|
|
|
352
351
|
|
|
353
352
|
@pytest.fixture(scope="session")
|
|
354
|
-
def
|
|
353
|
+
def constants_nor_polarimetric() -> VispConstantsDb:
|
|
355
354
|
return VispConstantsDb(
|
|
356
355
|
SOLAR_EXPOSURE_TIMES=(10.0,),
|
|
357
356
|
OBSERVE_EXPOSURE_TIMES=(10.0,),
|
|
@@ -361,42 +360,47 @@ def constatns_nor_polarimetric() -> VispConstantsDb:
|
|
|
361
360
|
|
|
362
361
|
|
|
363
362
|
@pytest.fixture(scope="function")
|
|
364
|
-
def
|
|
363
|
+
def background_light_calibration_task_factory(
|
|
365
364
|
tmp_path,
|
|
366
|
-
assign_input_dataset_doc_to_task,
|
|
367
365
|
init_visp_constants_db,
|
|
366
|
+
default_constants,
|
|
368
367
|
recipe_run_id,
|
|
368
|
+
assign_input_dataset_doc_to_task,
|
|
369
369
|
background_testing_parameter_values,
|
|
370
370
|
):
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
371
|
+
def make_task(background_on: bool = True, constants: VispConstantsDb = default_constants):
|
|
372
|
+
constants_db = VispConstantsDb()
|
|
373
|
+
init_visp_constants_db(recipe_run_id, constants_db)
|
|
374
|
+
with BackgroundLightCalibration(
|
|
375
|
+
recipe_run_id=recipe_run_id,
|
|
376
|
+
workflow_name="background_light_calibration",
|
|
377
|
+
workflow_version="vX.Y",
|
|
378
|
+
) as task:
|
|
379
|
+
try: # This try... block is here to make sure the dbs get cleaned up if there's a failure in the fixture
|
|
380
|
+
task.scratch = WorkflowFileSystem(
|
|
381
|
+
scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
|
|
382
|
+
)
|
|
383
|
+
init_visp_constants_db(task.recipe_run_id, constants)
|
|
384
|
+
assign_input_dataset_doc_to_task(
|
|
385
|
+
task, background_testing_parameter_values(background_on=background_on)
|
|
386
|
+
)
|
|
387
|
+
yield task
|
|
388
|
+
except:
|
|
389
|
+
raise
|
|
390
|
+
finally:
|
|
391
|
+
task._purge()
|
|
383
392
|
|
|
384
|
-
|
|
385
|
-
except:
|
|
386
|
-
raise
|
|
387
|
-
finally:
|
|
388
|
-
task._purge()
|
|
393
|
+
return make_task
|
|
389
394
|
|
|
390
395
|
|
|
391
396
|
def test_background_light_calibration_task(
|
|
392
|
-
|
|
397
|
+
background_light_calibration_task_factory,
|
|
393
398
|
write_background_input_files_to_task,
|
|
394
|
-
default_constants,
|
|
395
|
-
init_visp_constants_db,
|
|
396
399
|
total_polcal_files,
|
|
397
400
|
num_polcal_dark_files,
|
|
398
401
|
background_light,
|
|
399
402
|
mocker,
|
|
403
|
+
fake_gql_client,
|
|
400
404
|
):
|
|
401
405
|
"""
|
|
402
406
|
Give: a BackgroundLightCalibrationTask with a valid set of polcal data
|
|
@@ -404,10 +408,10 @@ def test_background_light_calibration_task(
|
|
|
404
408
|
Then: BACKGROUND intermediate frames are generated for each beam
|
|
405
409
|
"""
|
|
406
410
|
mocker.patch(
|
|
407
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
411
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
408
412
|
)
|
|
409
|
-
|
|
410
|
-
|
|
413
|
+
task_generator = background_light_calibration_task_factory()
|
|
414
|
+
task = next(task_generator)
|
|
411
415
|
write_background_input_files_to_task(task)
|
|
412
416
|
task()
|
|
413
417
|
|
|
@@ -435,7 +439,10 @@ def test_background_light_calibration_task(
|
|
|
435
439
|
|
|
436
440
|
|
|
437
441
|
def test_background_light_bad_exposure_times(
|
|
438
|
-
|
|
442
|
+
background_light_calibration_task_factory,
|
|
443
|
+
constants_bad_exp_times,
|
|
444
|
+
mocker,
|
|
445
|
+
fake_gql_client,
|
|
439
446
|
):
|
|
440
447
|
"""
|
|
441
448
|
Given: A set of data where the polcal exposure times don't match those of solar gain and observe
|
|
@@ -443,16 +450,19 @@ def test_background_light_bad_exposure_times(
|
|
|
443
450
|
Then: An error is raised
|
|
444
451
|
"""
|
|
445
452
|
mocker.patch(
|
|
446
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
453
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
447
454
|
)
|
|
448
|
-
|
|
449
|
-
|
|
455
|
+
task_generator = background_light_calibration_task_factory(constants=constants_bad_exp_times)
|
|
456
|
+
task = next(task_generator)
|
|
450
457
|
with pytest.raises(ValueError, match="do not all have the same FPA exposure time"):
|
|
451
458
|
task()
|
|
452
459
|
|
|
453
460
|
|
|
454
461
|
def test_background_light_non_polarimetric_dataset(
|
|
455
|
-
|
|
462
|
+
background_light_calibration_task_factory,
|
|
463
|
+
constants_nor_polarimetric,
|
|
464
|
+
mocker,
|
|
465
|
+
fake_gql_client,
|
|
456
466
|
):
|
|
457
467
|
"""
|
|
458
468
|
Given: A dataset that is non-polarimetric (i.e., Stokes-I only)
|
|
@@ -460,10 +470,11 @@ def test_background_light_non_polarimetric_dataset(
|
|
|
460
470
|
Then: Nothing is done and no files are written
|
|
461
471
|
"""
|
|
462
472
|
mocker.patch(
|
|
463
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
473
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
464
474
|
)
|
|
465
|
-
|
|
466
|
-
|
|
475
|
+
|
|
476
|
+
task_generator = background_light_calibration_task_factory(constants=constants_nor_polarimetric)
|
|
477
|
+
task = next(task_generator)
|
|
467
478
|
task()
|
|
468
479
|
|
|
469
480
|
# Test that no BACKGROUND files were created
|
|
@@ -474,12 +485,9 @@ def test_background_light_non_polarimetric_dataset(
|
|
|
474
485
|
|
|
475
486
|
|
|
476
487
|
def test_background_light_switch_off(
|
|
477
|
-
|
|
478
|
-
default_constants,
|
|
479
|
-
init_visp_constants_db,
|
|
480
|
-
assign_input_dataset_doc_to_task,
|
|
481
|
-
background_testing_parameter_values,
|
|
488
|
+
background_light_calibration_task_factory,
|
|
482
489
|
mocker,
|
|
490
|
+
fake_gql_client,
|
|
483
491
|
):
|
|
484
492
|
"""
|
|
485
493
|
Given: A task with the background light switch turned off
|
|
@@ -487,11 +495,10 @@ def test_background_light_switch_off(
|
|
|
487
495
|
Then: Nothing is done and no files are written
|
|
488
496
|
"""
|
|
489
497
|
mocker.patch(
|
|
490
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
498
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
491
499
|
)
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
assign_input_dataset_doc_to_task(task, background_testing_parameter_values(background_on=False))
|
|
500
|
+
task_generator = background_light_calibration_task_factory(background_on=False)
|
|
501
|
+
task = next(task_generator)
|
|
495
502
|
task()
|
|
496
503
|
|
|
497
504
|
# Test that no BACKGROUND files were created
|
|
@@ -6,7 +6,6 @@ import pytest
|
|
|
6
6
|
from astropy.io import fits
|
|
7
7
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
8
8
|
from dkist_processing_common.models.tags import Tag
|
|
9
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
10
9
|
|
|
11
10
|
from dkist_processing_visp.models.tags import VispTag
|
|
12
11
|
from dkist_processing_visp.tasks.dark import DarkCalibration
|
|
@@ -90,14 +89,16 @@ def dark_calibration_task(tmp_path, init_visp_constants_db, recipe_run_id):
|
|
|
90
89
|
task._purge()
|
|
91
90
|
|
|
92
91
|
|
|
93
|
-
def test_dark_calibration_task(
|
|
92
|
+
def test_dark_calibration_task(
|
|
93
|
+
dark_calibration_task, assign_input_dataset_doc_to_task, mocker, fake_gql_client
|
|
94
|
+
):
|
|
94
95
|
"""
|
|
95
96
|
Given: A DarkCalibration task with multiple task exposure times
|
|
96
97
|
When: Calling the task instance
|
|
97
98
|
Then: Only one average intermediate dark frame exists for each exposure time and unused times are not made
|
|
98
99
|
"""
|
|
99
100
|
mocker.patch(
|
|
100
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
101
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
101
102
|
)
|
|
102
103
|
# Given
|
|
103
104
|
task, readout_exp_times = dark_calibration_task
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from dkist_header_validator.translator import translate_spec122_to_spec214_l0
|
|
3
|
+
|
|
4
|
+
from dkist_processing_visp.models.fits_access import VispMetadataKey
|
|
5
|
+
from dkist_processing_visp.parsers.visp_l0_fits_access import VispL0FitsAccess
|
|
6
|
+
from dkist_processing_visp.parsers.visp_l1_fits_access import VispL1FitsAccess
|
|
7
|
+
from dkist_processing_visp.tests.header_models import VispHeadersValidObserveFrames
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@pytest.fixture(scope="session")
|
|
11
|
+
def complete_header():
|
|
12
|
+
dataset = VispHeadersValidObserveFrames(
|
|
13
|
+
array_shape=(1, 1, 1),
|
|
14
|
+
time_delta=10,
|
|
15
|
+
num_maps=1,
|
|
16
|
+
num_raster_steps=1,
|
|
17
|
+
num_modstates=1,
|
|
18
|
+
)
|
|
19
|
+
header = translate_spec122_to_spec214_l0(dataset.header())
|
|
20
|
+
return header
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_metadata_keys_in_access_bases(complete_header):
|
|
24
|
+
"""
|
|
25
|
+
Given: the set of metadata key names in VispMetadataKey
|
|
26
|
+
When: the ViSP FITS access classes define a set of new attributes
|
|
27
|
+
Then: the sets are the same and the attributes have the correct values
|
|
28
|
+
"""
|
|
29
|
+
# Given
|
|
30
|
+
visp_metadata_key_names = {vmk.name for vmk in VispMetadataKey}
|
|
31
|
+
# When
|
|
32
|
+
all_visp_fits_access_attrs = set()
|
|
33
|
+
for access_class in [VispL0FitsAccess, VispL1FitsAccess]:
|
|
34
|
+
fits_obj = access_class.from_header(complete_header)
|
|
35
|
+
visp_instance_attrs = set(vars(fits_obj).keys())
|
|
36
|
+
parent_class = access_class.mro()[1]
|
|
37
|
+
parent_fits_obj = parent_class.from_header(complete_header)
|
|
38
|
+
parent_instance_attrs = set(vars(parent_fits_obj).keys())
|
|
39
|
+
visp_fits_access_attrs = visp_instance_attrs - parent_instance_attrs
|
|
40
|
+
for attr in visp_fits_access_attrs:
|
|
41
|
+
assert getattr(fits_obj, attr) == fits_obj.header[VispMetadataKey[attr]]
|
|
42
|
+
all_visp_fits_access_attrs |= visp_fits_access_attrs
|
|
43
|
+
assert visp_metadata_key_names == all_visp_fits_access_attrs
|
|
@@ -7,14 +7,13 @@ import pytest
|
|
|
7
7
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
8
8
|
from dkist_processing_common.codecs.fits import fits_array_decoder
|
|
9
9
|
from dkist_processing_common.models.tags import Tag
|
|
10
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
11
10
|
from dkist_processing_math import transform
|
|
12
11
|
|
|
13
12
|
from dkist_processing_visp.models.tags import VispTag
|
|
14
13
|
from dkist_processing_visp.tasks.geometric import GeometricCalibration
|
|
15
|
-
from dkist_processing_visp.tests.conftest import tag_on_modstate
|
|
16
14
|
from dkist_processing_visp.tests.conftest import VispConstantsDb
|
|
17
15
|
from dkist_processing_visp.tests.conftest import VispInputDatasetParameterValues
|
|
16
|
+
from dkist_processing_visp.tests.conftest import tag_on_modstate
|
|
18
17
|
from dkist_processing_visp.tests.conftest import write_frames_to_task
|
|
19
18
|
from dkist_processing_visp.tests.conftest import write_intermediate_darks_to_task
|
|
20
19
|
from dkist_processing_visp.tests.header_models import VispHeadersInputLampGainFrames
|
|
@@ -189,7 +188,9 @@ def geometric_calibration_task(tmp_path, recipe_run_id, init_visp_constants_db):
|
|
|
189
188
|
task._purge()
|
|
190
189
|
|
|
191
190
|
|
|
192
|
-
def test_geometric_task(
|
|
191
|
+
def test_geometric_task(
|
|
192
|
+
geometric_calibration_task, assign_input_dataset_doc_to_task, mocker, fake_gql_client
|
|
193
|
+
):
|
|
193
194
|
"""
|
|
194
195
|
Given: A set of raw solar gain images and necessary intermediate calibrations
|
|
195
196
|
When: Running the geometric task
|
|
@@ -201,7 +202,7 @@ def test_geometric_task(geometric_calibration_task, assign_input_dataset_doc_to_
|
|
|
201
202
|
# tests for that. In other words, this fixture just tests if the machinery of the task completes and some object
|
|
202
203
|
# (ANY object) is written correctly.
|
|
203
204
|
mocker.patch(
|
|
204
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
205
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
205
206
|
)
|
|
206
207
|
task, readout_exp_times, num_modstates = geometric_calibration_task
|
|
207
208
|
dark_signal = 3.0
|
|
@@ -357,3 +358,43 @@ def test_basic_corrections(geometric_calibration_task, assign_input_dataset_doc_
|
|
|
357
358
|
lamp_array = task.basic_corrected_lamp_data(beam=beam, modstate=modstate)
|
|
358
359
|
np.testing.assert_equal(expected, solar_array)
|
|
359
360
|
np.testing.assert_equal(expected, lamp_array)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def test_line_zones(geometric_calibration_task):
|
|
364
|
+
"""
|
|
365
|
+
Given: A spectrum with some absorption lines
|
|
366
|
+
When: Computing zones around the lines
|
|
367
|
+
Then: Correct results are returned
|
|
368
|
+
"""
|
|
369
|
+
|
|
370
|
+
# NOTE that it does not test for removal of overlapping regions
|
|
371
|
+
def gaussian(x, amp, mu, sig):
|
|
372
|
+
return amp * np.exp(-np.power(x - mu, 2.0) / (2 * np.power(sig, 2.0)))
|
|
373
|
+
|
|
374
|
+
spec = np.ones(1000) * 100
|
|
375
|
+
x = np.arange(1000.0)
|
|
376
|
+
expected = []
|
|
377
|
+
for m, s in zip([100.0, 300.0, 700], [10.0, 20.0, 5.0]):
|
|
378
|
+
spec -= gaussian(x, 40, m, s)
|
|
379
|
+
hwhm = s * 2.355 / 2
|
|
380
|
+
expected.append((np.floor(m - hwhm).astype(int), np.ceil(m + hwhm).astype(int)))
|
|
381
|
+
|
|
382
|
+
task = geometric_calibration_task[0]
|
|
383
|
+
|
|
384
|
+
zones = task.compute_line_zones(spec[:, None], bg_order=0, rel_height=0.5)
|
|
385
|
+
assert zones == expected
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def test_identify_overlapping_zones(geometric_calibration_task):
|
|
389
|
+
"""
|
|
390
|
+
Given: A list of zone borders that contain overlapping zones
|
|
391
|
+
When: Identifying zones that overlap
|
|
392
|
+
Then: The smaller of the overlapping zones are identified for removal
|
|
393
|
+
"""
|
|
394
|
+
rips = np.array([100, 110, 220, 200])
|
|
395
|
+
lips = np.array([150, 120, 230, 250])
|
|
396
|
+
|
|
397
|
+
task = geometric_calibration_task[0]
|
|
398
|
+
|
|
399
|
+
idx_to_remove = task.identify_overlapping_zones(rips, lips)
|
|
400
|
+
assert idx_to_remove == [1, 2]
|
|
@@ -6,7 +6,6 @@ import pytest
|
|
|
6
6
|
from astropy.io import fits
|
|
7
7
|
from dkist_data_simulator.spec122 import Spec122Dataset
|
|
8
8
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
9
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
10
9
|
from dkist_processing_pac.fitter.polcal_fitter import PolcalFitter
|
|
11
10
|
from dkist_processing_pac.input_data.dresser import Dresser
|
|
12
11
|
|
|
@@ -19,7 +18,11 @@ from dkist_processing_visp.tests.conftest import write_frames_to_task
|
|
|
19
18
|
from dkist_processing_visp.tests.conftest import write_intermediate_background_to_task
|
|
20
19
|
from dkist_processing_visp.tests.conftest import write_intermediate_darks_to_task
|
|
21
20
|
from dkist_processing_visp.tests.conftest import write_intermediate_geometric_to_task
|
|
21
|
+
from dkist_processing_visp.tests.conftest import write_intermediate_polcal_darks_to_task
|
|
22
|
+
from dkist_processing_visp.tests.conftest import write_intermediate_polcal_gains_to_task
|
|
23
|
+
from dkist_processing_visp.tests.header_models import VispHeadersInputPolcalDarkFrames
|
|
22
24
|
from dkist_processing_visp.tests.header_models import VispHeadersInputPolcalFrames
|
|
25
|
+
from dkist_processing_visp.tests.header_models import VispHeadersInputPolcalGainFrames
|
|
23
26
|
|
|
24
27
|
|
|
25
28
|
class DummyPolcalFitter(PolcalFitter):
|
|
@@ -88,6 +91,58 @@ def write_input_polcals_to_task(
|
|
|
88
91
|
)
|
|
89
92
|
|
|
90
93
|
|
|
94
|
+
def write_input_polcal_darks_to_task(
|
|
95
|
+
task,
|
|
96
|
+
readout_exp_time: float,
|
|
97
|
+
num_modstates: int,
|
|
98
|
+
data_shape: tuple[int, int],
|
|
99
|
+
):
|
|
100
|
+
array_shape = (1, *data_shape)
|
|
101
|
+
dataset = VispHeadersInputPolcalDarkFrames(
|
|
102
|
+
array_shape=array_shape,
|
|
103
|
+
time_delta=10,
|
|
104
|
+
num_modstates=num_modstates,
|
|
105
|
+
readout_exp_time=readout_exp_time,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
write_frames_to_task(
|
|
109
|
+
task=task,
|
|
110
|
+
frame_generator=dataset,
|
|
111
|
+
extra_tags=[
|
|
112
|
+
VispTag.input(),
|
|
113
|
+
VispTag.task_polcal(),
|
|
114
|
+
VispTag.task_polcal_dark(),
|
|
115
|
+
VispTag.readout_exp_time(readout_exp_time),
|
|
116
|
+
],
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def write_input_polcal_gains_to_task(
|
|
121
|
+
task,
|
|
122
|
+
readout_exp_time: float,
|
|
123
|
+
num_modstates: int,
|
|
124
|
+
data_shape: tuple[int, int],
|
|
125
|
+
):
|
|
126
|
+
array_shape = (1, *data_shape)
|
|
127
|
+
dataset = VispHeadersInputPolcalGainFrames(
|
|
128
|
+
array_shape=array_shape,
|
|
129
|
+
time_delta=10,
|
|
130
|
+
num_modstates=num_modstates,
|
|
131
|
+
readout_exp_time=readout_exp_time,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
write_frames_to_task(
|
|
135
|
+
task=task,
|
|
136
|
+
frame_generator=dataset,
|
|
137
|
+
extra_tags=[
|
|
138
|
+
VispTag.input(),
|
|
139
|
+
VispTag.task_polcal(),
|
|
140
|
+
VispTag.task_polcal_gain(),
|
|
141
|
+
VispTag.readout_exp_time(readout_exp_time),
|
|
142
|
+
],
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
|
|
91
146
|
@pytest.fixture(scope="function")
|
|
92
147
|
def instrument_polarization_calibration_task(
|
|
93
148
|
tmp_path,
|
|
@@ -157,6 +212,7 @@ def test_instrument_polarization_calibration_task(
|
|
|
157
212
|
background_on,
|
|
158
213
|
assign_input_dataset_doc_to_task,
|
|
159
214
|
mocker,
|
|
215
|
+
fake_gql_client,
|
|
160
216
|
):
|
|
161
217
|
"""
|
|
162
218
|
Given: An InstrumentPolarizationCalibration task
|
|
@@ -165,7 +221,7 @@ def test_instrument_polarization_calibration_task(
|
|
|
165
221
|
"""
|
|
166
222
|
|
|
167
223
|
mocker.patch(
|
|
168
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
224
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
169
225
|
)
|
|
170
226
|
mocker.patch(
|
|
171
227
|
"dkist_processing_visp.tasks.instrument_polarization.PolcalFitter",
|
|
@@ -190,12 +246,6 @@ def test_instrument_polarization_calibration_task(
|
|
|
190
246
|
),
|
|
191
247
|
)
|
|
192
248
|
|
|
193
|
-
write_intermediate_darks_to_task(
|
|
194
|
-
task=task,
|
|
195
|
-
dark_signal=0,
|
|
196
|
-
readout_exp_time=readout_exp_time,
|
|
197
|
-
data_shape=intermediate_shape,
|
|
198
|
-
)
|
|
199
249
|
if background_on:
|
|
200
250
|
write_intermediate_background_to_task(
|
|
201
251
|
task=task, background_signal=0.0, data_shape=intermediate_shape
|
|
@@ -205,7 +255,8 @@ def test_instrument_polarization_calibration_task(
|
|
|
205
255
|
task=task, num_modstates=num_modstates, data_shape=intermediate_shape
|
|
206
256
|
)
|
|
207
257
|
write_dummy_intermediate_solar_cals_to_task(
|
|
208
|
-
task=task,
|
|
258
|
+
task=task,
|
|
259
|
+
data_shape=intermediate_shape,
|
|
209
260
|
)
|
|
210
261
|
write_input_polcals_to_task(
|
|
211
262
|
task=task,
|
|
@@ -214,6 +265,18 @@ def test_instrument_polarization_calibration_task(
|
|
|
214
265
|
num_cs_steps=num_cs_steps,
|
|
215
266
|
data_shape=input_shape,
|
|
216
267
|
)
|
|
268
|
+
write_input_polcal_darks_to_task(
|
|
269
|
+
task=task,
|
|
270
|
+
readout_exp_time=readout_exp_time,
|
|
271
|
+
num_modstates=num_modstates,
|
|
272
|
+
data_shape=input_shape,
|
|
273
|
+
)
|
|
274
|
+
write_input_polcal_gains_to_task(
|
|
275
|
+
task=task,
|
|
276
|
+
readout_exp_time=readout_exp_time,
|
|
277
|
+
num_modstates=num_modstates,
|
|
278
|
+
data_shape=input_shape,
|
|
279
|
+
)
|
|
217
280
|
|
|
218
281
|
task()
|
|
219
282
|
|
|
@@ -6,13 +6,12 @@ import pytest
|
|
|
6
6
|
from astropy.io import fits
|
|
7
7
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
8
8
|
from dkist_processing_common.models.tags import Tag
|
|
9
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
10
9
|
|
|
11
10
|
from dkist_processing_visp.models.tags import VispTag
|
|
12
11
|
from dkist_processing_visp.tasks.lamp import LampCalibration
|
|
13
|
-
from dkist_processing_visp.tests.conftest import tag_on_modstate
|
|
14
12
|
from dkist_processing_visp.tests.conftest import VispConstantsDb
|
|
15
13
|
from dkist_processing_visp.tests.conftest import VispInputDatasetParameterValues
|
|
14
|
+
from dkist_processing_visp.tests.conftest import tag_on_modstate
|
|
16
15
|
from dkist_processing_visp.tests.conftest import write_frames_to_task
|
|
17
16
|
from dkist_processing_visp.tests.conftest import write_intermediate_darks_to_task
|
|
18
17
|
from dkist_processing_visp.tests.header_models import VispHeadersInputLampGainFrames
|
|
@@ -24,9 +23,10 @@ def make_lamp_array_data(
|
|
|
24
23
|
frame: VispHeadersInputLampGainFrames, dark_signal: float, beam_border: int
|
|
25
24
|
):
|
|
26
25
|
num_raw_frames_per_fpa = frame.header()["CAM__014"]
|
|
26
|
+
modstate = frame.current_modstate("") # Weird signature due to @key_function
|
|
27
27
|
data = np.zeros(frame.array_shape)
|
|
28
|
-
data[0, :beam_border, :] = (1.1 + dark_signal) * num_raw_frames_per_fpa
|
|
29
|
-
data[0, beam_border:, :] = (
|
|
28
|
+
data[0, :beam_border, :] = (1.0 + 0.1 * modstate + dark_signal) * num_raw_frames_per_fpa
|
|
29
|
+
data[0, beam_border:, :] = (2.0 + 0.1 * modstate + dark_signal) * num_raw_frames_per_fpa
|
|
30
30
|
|
|
31
31
|
return data
|
|
32
32
|
|
|
@@ -91,14 +91,16 @@ def lamp_calibration_task(
|
|
|
91
91
|
task._purge()
|
|
92
92
|
|
|
93
93
|
|
|
94
|
-
def test_lamp_calibration_task(
|
|
94
|
+
def test_lamp_calibration_task(
|
|
95
|
+
lamp_calibration_task, assign_input_dataset_doc_to_task, mocker, fake_gql_client
|
|
96
|
+
):
|
|
95
97
|
"""
|
|
96
98
|
Given: A LampCalibration task
|
|
97
99
|
When: Calling the task instance
|
|
98
100
|
Then: The correct number of output lamp gain frames exists, and are tagged correctly
|
|
99
101
|
"""
|
|
100
102
|
mocker.patch(
|
|
101
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
103
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
102
104
|
)
|
|
103
105
|
# Given
|
|
104
106
|
task, num_modstates, readout_exp_time = lamp_calibration_task
|
|
@@ -131,27 +133,21 @@ def test_lamp_calibration_task(lamp_calibration_task, assign_input_dataset_doc_t
|
|
|
131
133
|
VispTag.task_lamp_gain(),
|
|
132
134
|
VispTag.intermediate(),
|
|
133
135
|
]
|
|
134
|
-
assert len(list(task.read(tags=tags))) ==
|
|
135
|
-
|
|
136
|
-
for i in range(num_modstates):
|
|
137
|
-
for beam in [1, 2]:
|
|
138
|
-
tags = [
|
|
139
|
-
VispTag.task_lamp_gain(),
|
|
140
|
-
VispTag.intermediate(),
|
|
141
|
-
VispTag.modstate(i + 1),
|
|
142
|
-
VispTag.beam(beam),
|
|
143
|
-
]
|
|
144
|
-
files = list(task.read(tags=tags))
|
|
145
|
-
assert len(files) == 1
|
|
146
|
-
hdu = fits.open(files[0])[0]
|
|
147
|
-
np.testing.assert_allclose(hdu.data, np.ones((10, 10)) * (1 + (0.1 * beam)))
|
|
136
|
+
assert len(list(task.read(tags=tags))) == 2 # One per beam
|
|
148
137
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
138
|
+
for beam in [1, 2]:
|
|
139
|
+
tags = [
|
|
140
|
+
VispTag.task_lamp_gain(),
|
|
141
|
+
VispTag.intermediate(),
|
|
142
|
+
VispTag.beam(beam),
|
|
143
|
+
]
|
|
144
|
+
files = list(task.read(tags=tags))
|
|
145
|
+
assert len(files) == 1
|
|
146
|
+
|
|
147
|
+
expected_signal = beam + np.mean(np.arange(1, num_modstates + 1)) * 0.1
|
|
148
|
+
|
|
149
|
+
hdu = fits.open(files[0])[0]
|
|
150
|
+
np.testing.assert_allclose(hdu.data, np.ones(intermediate_shape) * expected_signal)
|
|
155
151
|
|
|
156
152
|
quality_files = task.read(tags=[Tag.quality("TASK_TYPES")])
|
|
157
153
|
for file in quality_files:
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import pytest
|
|
2
2
|
from astropy.io import fits
|
|
3
3
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
4
|
-
from dkist_processing_common.
|
|
4
|
+
from dkist_processing_common.models.fits_access import MetadataKey
|
|
5
5
|
|
|
6
6
|
from dkist_processing_visp.models.tags import VispTag
|
|
7
7
|
from dkist_processing_visp.tasks.make_movie_frames import MakeVispMovieFrames
|
|
@@ -30,14 +30,14 @@ def movie_frames_task(tmp_path, recipe_run_id, init_visp_constants_db):
|
|
|
30
30
|
|
|
31
31
|
|
|
32
32
|
@pytest.mark.parametrize("pol_mode", ["observe_polarimetric", "observe_intensity"])
|
|
33
|
-
def test_make_movie_frames(movie_frames_task, pol_mode, mocker):
|
|
33
|
+
def test_make_movie_frames(movie_frames_task, pol_mode, mocker, fake_gql_client):
|
|
34
34
|
"""
|
|
35
35
|
Given: A MakeVispMovieFrames task
|
|
36
36
|
When: Calling the task instance
|
|
37
37
|
Then: a fits file is made for each raster scan containing the movie frame for that scan
|
|
38
38
|
"""
|
|
39
39
|
mocker.patch(
|
|
40
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
40
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
41
41
|
)
|
|
42
42
|
task, num_maps, num_steps = movie_frames_task
|
|
43
43
|
spectral_size = 3
|
|
@@ -57,5 +57,5 @@ def test_make_movie_frames(movie_frames_task, pol_mode, mocker):
|
|
|
57
57
|
for filepath in task.read(tags=[VispTag.movie_frame()]):
|
|
58
58
|
assert filepath.exists()
|
|
59
59
|
hdul = fits.open(filepath)
|
|
60
|
-
assert hdul[0].header[
|
|
60
|
+
assert hdul[0].header[MetadataKey.instrument] == "VISP"
|
|
61
61
|
assert hdul[0].data.shape == expected_movie_fram_shape
|
|
@@ -14,6 +14,7 @@ from dkist_processing_common.parsers.single_value_single_key_flower import (
|
|
|
14
14
|
)
|
|
15
15
|
|
|
16
16
|
from dkist_processing_visp.models.constants import VispBudName
|
|
17
|
+
from dkist_processing_visp.models.fits_access import VispMetadataKey
|
|
17
18
|
from dkist_processing_visp.models.tags import VispStemName
|
|
18
19
|
from dkist_processing_visp.models.tags import VispTag
|
|
19
20
|
from dkist_processing_visp.parsers.map_repeats import MapScanFlower
|
|
@@ -90,7 +91,8 @@ class ParseTaskJustMapStuff(ParseL0VispInputData):
|
|
|
90
91
|
MapScanFlower(),
|
|
91
92
|
RasterScanStepFlower(),
|
|
92
93
|
SingleValueSingleKeyFlower(
|
|
93
|
-
tag_stem_name=VispStemName.modstate.value,
|
|
94
|
+
tag_stem_name=VispStemName.modstate.value,
|
|
95
|
+
metadata_key=VispMetadataKey.modulator_state,
|
|
94
96
|
),
|
|
95
97
|
]
|
|
96
98
|
|