dkist-processing-cryonirsp 1.4.20__py3-none-any.whl → 1.14.9rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- changelog/232.misc.rst +1 -0
- dkist_processing_cryonirsp/__init__.py +1 -0
- dkist_processing_cryonirsp/codecs/fits.py +1 -0
- dkist_processing_cryonirsp/config.py +5 -1
- dkist_processing_cryonirsp/models/beam_boundaries.py +1 -0
- dkist_processing_cryonirsp/models/constants.py +31 -30
- dkist_processing_cryonirsp/models/exposure_conditions.py +6 -5
- dkist_processing_cryonirsp/models/fits_access.py +40 -0
- dkist_processing_cryonirsp/models/parameters.py +14 -26
- dkist_processing_cryonirsp/models/tags.py +1 -0
- dkist_processing_cryonirsp/models/task_name.py +1 -0
- dkist_processing_cryonirsp/parsers/check_for_gains.py +1 -0
- dkist_processing_cryonirsp/parsers/cryonirsp_l0_fits_access.py +40 -47
- dkist_processing_cryonirsp/parsers/cryonirsp_l1_fits_access.py +1 -0
- dkist_processing_cryonirsp/parsers/exposure_conditions.py +14 -13
- dkist_processing_cryonirsp/parsers/map_repeats.py +1 -0
- dkist_processing_cryonirsp/parsers/measurements.py +29 -16
- dkist_processing_cryonirsp/parsers/modstates.py +5 -1
- dkist_processing_cryonirsp/parsers/optical_density_filters.py +1 -0
- dkist_processing_cryonirsp/parsers/polarimetric_check.py +18 -7
- dkist_processing_cryonirsp/parsers/scan_step.py +12 -4
- dkist_processing_cryonirsp/parsers/time.py +7 -7
- dkist_processing_cryonirsp/parsers/wavelength.py +6 -1
- dkist_processing_cryonirsp/tasks/__init__.py +2 -1
- dkist_processing_cryonirsp/tasks/assemble_movie.py +1 -0
- dkist_processing_cryonirsp/tasks/bad_pixel_map.py +6 -5
- dkist_processing_cryonirsp/tasks/beam_boundaries_base.py +12 -11
- dkist_processing_cryonirsp/tasks/ci_beam_boundaries.py +1 -0
- dkist_processing_cryonirsp/tasks/ci_science.py +1 -0
- dkist_processing_cryonirsp/tasks/cryonirsp_base.py +2 -3
- dkist_processing_cryonirsp/tasks/dark.py +5 -4
- dkist_processing_cryonirsp/tasks/gain.py +7 -6
- dkist_processing_cryonirsp/tasks/instrument_polarization.py +17 -16
- dkist_processing_cryonirsp/tasks/l1_output_data.py +1 -0
- dkist_processing_cryonirsp/tasks/linearity_correction.py +1 -0
- dkist_processing_cryonirsp/tasks/make_movie_frames.py +3 -2
- dkist_processing_cryonirsp/tasks/mixin/corrections.py +1 -0
- dkist_processing_cryonirsp/tasks/mixin/shift_measurements.py +9 -2
- dkist_processing_cryonirsp/tasks/parse.py +70 -52
- dkist_processing_cryonirsp/tasks/quality_metrics.py +15 -14
- dkist_processing_cryonirsp/tasks/science_base.py +8 -6
- dkist_processing_cryonirsp/tasks/sp_beam_boundaries.py +2 -1
- dkist_processing_cryonirsp/tasks/sp_geometric.py +11 -10
- dkist_processing_cryonirsp/tasks/sp_science.py +1 -0
- dkist_processing_cryonirsp/tasks/sp_solar_gain.py +15 -12
- dkist_processing_cryonirsp/tasks/sp_wavelength_calibration.py +300 -0
- dkist_processing_cryonirsp/tasks/write_l1.py +59 -38
- dkist_processing_cryonirsp/tests/conftest.py +75 -53
- dkist_processing_cryonirsp/tests/header_models.py +62 -11
- dkist_processing_cryonirsp/tests/local_trial_workflows/l0_cals_only.py +26 -46
- dkist_processing_cryonirsp/tests/local_trial_workflows/l0_to_l1.py +26 -47
- dkist_processing_cryonirsp/tests/local_trial_workflows/linearize_only.py +3 -3
- dkist_processing_cryonirsp/tests/local_trial_workflows/local_trial_helpers.py +57 -26
- dkist_processing_cryonirsp/tests/test_assemble_movie.py +4 -5
- dkist_processing_cryonirsp/tests/test_assemble_qualilty.py +5 -1
- dkist_processing_cryonirsp/tests/test_bad_pixel_maps.py +4 -5
- dkist_processing_cryonirsp/tests/test_ci_beam_boundaries.py +4 -5
- dkist_processing_cryonirsp/tests/test_ci_science.py +4 -5
- dkist_processing_cryonirsp/tests/test_corrections.py +5 -6
- dkist_processing_cryonirsp/tests/test_cryo_base.py +4 -6
- dkist_processing_cryonirsp/tests/test_cryo_constants.py +7 -3
- dkist_processing_cryonirsp/tests/test_dark.py +7 -8
- dkist_processing_cryonirsp/tests/test_fits_access.py +44 -0
- dkist_processing_cryonirsp/tests/test_gain.py +7 -8
- dkist_processing_cryonirsp/tests/test_instrument_polarization.py +19 -10
- dkist_processing_cryonirsp/tests/test_linearity_correction.py +5 -4
- dkist_processing_cryonirsp/tests/test_make_movie_frames.py +2 -3
- dkist_processing_cryonirsp/tests/test_parameters.py +23 -28
- dkist_processing_cryonirsp/tests/test_parse.py +48 -12
- dkist_processing_cryonirsp/tests/test_quality.py +2 -3
- dkist_processing_cryonirsp/tests/test_sp_beam_boundaries.py +5 -5
- dkist_processing_cryonirsp/tests/test_sp_geometric.py +5 -6
- dkist_processing_cryonirsp/tests/test_sp_make_movie_frames.py +2 -3
- dkist_processing_cryonirsp/tests/test_sp_science.py +4 -5
- dkist_processing_cryonirsp/tests/test_sp_solar.py +6 -5
- dkist_processing_cryonirsp/tests/{test_sp_dispersion_axis_correction.py → test_sp_wavelength_calibration.py} +11 -29
- dkist_processing_cryonirsp/tests/test_trial_create_quality_report.py +1 -1
- dkist_processing_cryonirsp/tests/test_workflows.py +1 -0
- dkist_processing_cryonirsp/tests/test_write_l1.py +29 -31
- dkist_processing_cryonirsp/workflows/__init__.py +1 -0
- dkist_processing_cryonirsp/workflows/ci_l0_processing.py +9 -5
- dkist_processing_cryonirsp/workflows/sp_l0_processing.py +12 -8
- dkist_processing_cryonirsp/workflows/trial_workflows.py +12 -11
- dkist_processing_cryonirsp-1.14.9rc1.dist-info/METADATA +552 -0
- dkist_processing_cryonirsp-1.14.9rc1.dist-info/RECORD +115 -0
- {dkist_processing_cryonirsp-1.4.20.dist-info → dkist_processing_cryonirsp-1.14.9rc1.dist-info}/WHEEL +1 -1
- docs/ci_science_calibration.rst +10 -0
- docs/conf.py +1 -0
- docs/index.rst +1 -0
- docs/sp_science_calibration.rst +7 -0
- docs/wavelength_calibration.rst +62 -0
- dkist_processing_cryonirsp/tasks/sp_dispersion_axis_correction.py +0 -492
- dkist_processing_cryonirsp-1.4.20.dist-info/METADATA +0 -452
- dkist_processing_cryonirsp-1.4.20.dist-info/RECORD +0 -111
- {dkist_processing_cryonirsp-1.4.20.dist-info → dkist_processing_cryonirsp-1.14.9rc1.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
"""Copy of UniqueBud from common that only activates if the frames are polarimetric "observe" or "polcal" task, or non-polarimetric "observe" task."""
|
|
2
|
-
|
|
2
|
+
|
|
3
|
+
from enum import StrEnum
|
|
4
|
+
from typing import NamedTuple
|
|
3
5
|
from typing import Type
|
|
4
6
|
|
|
5
7
|
from dkist_processing_common.models.flower_pot import SpilledDirt
|
|
@@ -7,21 +9,30 @@ from dkist_processing_common.models.flower_pot import Stem
|
|
|
7
9
|
from dkist_processing_common.models.flower_pot import Thorn
|
|
8
10
|
from dkist_processing_common.models.task_name import TaskName
|
|
9
11
|
|
|
10
|
-
from dkist_processing_cryonirsp.models.
|
|
12
|
+
from dkist_processing_cryonirsp.models.fits_access import CryonirspMetadataKey
|
|
11
13
|
from dkist_processing_cryonirsp.parsers.cryonirsp_l0_fits_access import CryonirspL0FitsAccess
|
|
12
14
|
|
|
13
15
|
|
|
16
|
+
class PolarimetricValueDataContainer(NamedTuple):
|
|
17
|
+
"""Named tuple to hold polarimetric metadata about a task."""
|
|
18
|
+
|
|
19
|
+
task: str
|
|
20
|
+
num_modstates: int
|
|
21
|
+
spin_mode: str
|
|
22
|
+
bud_value: str | int | float | bool
|
|
23
|
+
|
|
24
|
+
|
|
14
25
|
class PolarimetricCheckingUniqueBud(Stem):
|
|
15
26
|
"""Bud for checking if frames are polarimetric."""
|
|
16
27
|
|
|
17
|
-
PolarimetricValueData =
|
|
18
|
-
"PolarimetricValueData", ["task", "num_modstates", "spin_mode", "bud_value"]
|
|
19
|
-
)
|
|
28
|
+
PolarimetricValueData = PolarimetricValueDataContainer
|
|
20
29
|
observe_task_name = TaskName.observe.value.casefold()
|
|
21
30
|
polcal_task_name = TaskName.polcal.value.casefold()
|
|
22
31
|
|
|
23
|
-
def __init__(self, constant_name: str, metadata_key: str):
|
|
32
|
+
def __init__(self, constant_name: str, metadata_key: str | StrEnum):
|
|
24
33
|
super().__init__(stem_name=constant_name)
|
|
34
|
+
if isinstance(metadata_key, StrEnum):
|
|
35
|
+
metadata_key = metadata_key.name
|
|
25
36
|
self.metadata_key = metadata_key
|
|
26
37
|
|
|
27
38
|
@property
|
|
@@ -71,7 +82,7 @@ class PolarimetricCheckingUniqueBud(Stem):
|
|
|
71
82
|
# Some intensity mode data has the number of modulator states set to 0
|
|
72
83
|
num_modstates = fits_obj.number_of_modulator_states or 1
|
|
73
84
|
|
|
74
|
-
if self.metadata_key ==
|
|
85
|
+
if self.metadata_key == CryonirspMetadataKey.number_of_modulator_states.name:
|
|
75
86
|
bud_value = num_modstates
|
|
76
87
|
else:
|
|
77
88
|
bud_value = getattr(fits_obj, self.metadata_key)
|
|
@@ -3,6 +3,7 @@ Machinery for sorting files based on scan step.
|
|
|
3
3
|
|
|
4
4
|
Also includes base classes that can be used to sort files based on map scan.
|
|
5
5
|
"""
|
|
6
|
+
|
|
6
7
|
######################
|
|
7
8
|
# HOW THIS ALL WORKS #
|
|
8
9
|
######################
|
|
@@ -312,11 +313,11 @@ class MapScanStepStemBase(Stem, ABC):
|
|
|
312
313
|
return step_list.index(scan_step_obj) + 1 # Here we decide that map scan indices start at 1
|
|
313
314
|
|
|
314
315
|
|
|
315
|
-
class
|
|
316
|
-
"""
|
|
316
|
+
class NumberOfScanStepsBase(MapScanStepStemBase, ABC):
|
|
317
|
+
"""Base class for managing scan steps."""
|
|
317
318
|
|
|
318
|
-
def __init__(self):
|
|
319
|
-
super().__init__(stem_name=
|
|
319
|
+
def __init__(self, stem_name: CryonirspBudName):
|
|
320
|
+
super().__init__(stem_name=stem_name)
|
|
320
321
|
|
|
321
322
|
@cached_property
|
|
322
323
|
def map_scan_to_obj_dict(self) -> dict[int, list[SingleScanStep]]:
|
|
@@ -361,6 +362,13 @@ class NumberOfScanStepsBud(MapScanStepStemBase):
|
|
|
361
362
|
|
|
362
363
|
return len(completed_steps)
|
|
363
364
|
|
|
365
|
+
|
|
366
|
+
class NumberOfScanStepsBud(NumberOfScanStepsBase):
|
|
367
|
+
"""Bud for finding the total number of scan steps."""
|
|
368
|
+
|
|
369
|
+
def __init__(self):
|
|
370
|
+
super().__init__(stem_name=CryonirspBudName.num_scan_steps.value)
|
|
371
|
+
|
|
364
372
|
def getter(self, key):
|
|
365
373
|
"""
|
|
366
374
|
Compute the number of complete scan steps.
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Buds to parse exposure time."""
|
|
2
|
+
|
|
2
3
|
from typing import Hashable
|
|
3
4
|
from typing import Type
|
|
4
5
|
|
|
@@ -12,12 +13,11 @@ from dkist_processing_cryonirsp.parsers.cryonirsp_l0_fits_access import Cryonirs
|
|
|
12
13
|
from dkist_processing_cryonirsp.parsers.cryonirsp_l0_fits_access import CryonirspRampFitsAccess
|
|
13
14
|
|
|
14
15
|
|
|
15
|
-
class
|
|
16
|
-
"""Bud for finding the start time of the solar gain."""
|
|
16
|
+
class CryonirspSolarGainTimeObsBud(Stem):
|
|
17
|
+
"""Bud for finding the start time of the solar gain from the time_obs header key."""
|
|
17
18
|
|
|
18
19
|
def __init__(self):
|
|
19
|
-
|
|
20
|
-
super().__init__(stem_name=CryonirspBudName.solar_gain_ip_start_time.value)
|
|
20
|
+
super().__init__(stem_name=CryonirspBudName.solar_gain_start_time.value)
|
|
21
21
|
|
|
22
22
|
def setter(self, fits_obj: CryonirspL0FitsAccess) -> Type[SpilledDirt] | int:
|
|
23
23
|
"""
|
|
@@ -30,12 +30,12 @@ class CryonirspSolarGainStartTimeBud(Stem):
|
|
|
30
30
|
"""
|
|
31
31
|
if parse_header_ip_task_with_gains(fits_obj) != TaskName.solar_gain.value:
|
|
32
32
|
return SpilledDirt
|
|
33
|
-
return
|
|
33
|
+
return fits_obj.time_obs
|
|
34
34
|
|
|
35
35
|
def getter(self, key: Hashable):
|
|
36
36
|
"""Return the first date-obs value."""
|
|
37
|
-
|
|
38
|
-
return
|
|
37
|
+
first_time_obs = min(list(self.key_to_petal_dict.values()))
|
|
38
|
+
return first_time_obs
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
class CryonirspTimeObsBud(Stem):
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
"""CryoNIRSP bud to get the wavelength."""
|
|
2
|
+
|
|
3
|
+
from dkist_processing_common.models.fits_access import MetadataKey
|
|
2
4
|
from dkist_processing_common.models.flower_pot import SpilledDirt
|
|
3
5
|
from dkist_processing_common.parsers.unique_bud import UniqueBud
|
|
4
6
|
|
|
@@ -10,7 +12,10 @@ class ObserveWavelengthBud(UniqueBud):
|
|
|
10
12
|
"""Bud to find the wavelength."""
|
|
11
13
|
|
|
12
14
|
def __init__(self):
|
|
13
|
-
super().__init__(
|
|
15
|
+
super().__init__(
|
|
16
|
+
constant_name=CryonirspBudName.wavelength.value,
|
|
17
|
+
metadata_key=MetadataKey.wavelength,
|
|
18
|
+
)
|
|
14
19
|
|
|
15
20
|
def setter(self, fits_obj: CryonirspL0FitsAccess):
|
|
16
21
|
"""
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Init."""
|
|
2
|
+
|
|
2
3
|
from dkist_processing_cryonirsp.tasks.assemble_movie import *
|
|
3
4
|
from dkist_processing_cryonirsp.tasks.bad_pixel_map import *
|
|
4
5
|
from dkist_processing_cryonirsp.tasks.ci_beam_boundaries import *
|
|
@@ -11,8 +12,8 @@ from dkist_processing_cryonirsp.tasks.make_movie_frames import *
|
|
|
11
12
|
from dkist_processing_cryonirsp.tasks.parse import *
|
|
12
13
|
from dkist_processing_cryonirsp.tasks.quality_metrics import *
|
|
13
14
|
from dkist_processing_cryonirsp.tasks.sp_beam_boundaries import *
|
|
14
|
-
from dkist_processing_cryonirsp.tasks.sp_dispersion_axis_correction import *
|
|
15
15
|
from dkist_processing_cryonirsp.tasks.sp_geometric import *
|
|
16
16
|
from dkist_processing_cryonirsp.tasks.sp_science import *
|
|
17
17
|
from dkist_processing_cryonirsp.tasks.sp_solar_gain import *
|
|
18
|
+
from dkist_processing_cryonirsp.tasks.sp_wavelength_calibration import *
|
|
18
19
|
from dkist_processing_cryonirsp.tasks.write_l1 import *
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Task class for bad pixel map computation."""
|
|
2
|
+
|
|
2
3
|
import numpy as np
|
|
3
4
|
import scipy.ndimage as spnd
|
|
4
5
|
from dkist_processing_common.codecs.fits import fits_array_encoder
|
|
@@ -48,11 +49,11 @@ class BadPixelMapCalibration(CryonirspTaskBase):
|
|
|
48
49
|
None
|
|
49
50
|
|
|
50
51
|
"""
|
|
51
|
-
with self.
|
|
52
|
+
with self.telemetry_span(f"Compute average uncorrected solar gain image"):
|
|
52
53
|
average_solar_gain_array = self.compute_average_gain_array()
|
|
53
54
|
|
|
54
|
-
with self.
|
|
55
|
-
with self.
|
|
55
|
+
with self.telemetry_span(f"Compute the bad pixel map"):
|
|
56
|
+
with self.telemetry_span("Smooth array with median filter"):
|
|
56
57
|
filter_size = self.parameters.bad_pixel_map_median_filter_size
|
|
57
58
|
filtered_array = spnd.median_filter(
|
|
58
59
|
average_solar_gain_array,
|
|
@@ -61,7 +62,7 @@ class BadPixelMapCalibration(CryonirspTaskBase):
|
|
|
61
62
|
cval=np.nanmedian(average_solar_gain_array),
|
|
62
63
|
)
|
|
63
64
|
|
|
64
|
-
with self.
|
|
65
|
+
with self.telemetry_span("Identify bad pixels"):
|
|
65
66
|
thresh = self.parameters.bad_pixel_map_threshold_factor
|
|
66
67
|
|
|
67
68
|
diff = filtered_array - average_solar_gain_array
|
|
@@ -71,7 +72,7 @@ class BadPixelMapCalibration(CryonirspTaskBase):
|
|
|
71
72
|
zeros = np.where(average_solar_gain_array == 0.0)
|
|
72
73
|
bad_pixel_map[zeros] = 1
|
|
73
74
|
|
|
74
|
-
with self.
|
|
75
|
+
with self.telemetry_span("Writing bad pixel map"):
|
|
75
76
|
self.write(
|
|
76
77
|
data=bad_pixel_map,
|
|
77
78
|
tags=[CryonirspTag.intermediate_frame(), CryonirspTag.task_bad_pixel_map()],
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""CryoNIRSP compute beam boundary task."""
|
|
2
|
+
|
|
2
3
|
from abc import abstractmethod
|
|
3
4
|
|
|
4
5
|
import numpy as np
|
|
@@ -61,11 +62,11 @@ class BeamBoundariesCalibrationBase(CryonirspTaskBase):
|
|
|
61
62
|
|
|
62
63
|
"""
|
|
63
64
|
# Step 1:
|
|
64
|
-
with self.
|
|
65
|
+
with self.telemetry_span(f"Compute average solar gain image"):
|
|
65
66
|
average_solar_gain_array = self.compute_average_gain_array()
|
|
66
67
|
|
|
67
68
|
# Step 2:
|
|
68
|
-
with self.
|
|
69
|
+
with self.telemetry_span(f"Retrieve bad pixel map"):
|
|
69
70
|
bad_pixel_map = next(
|
|
70
71
|
self.read(
|
|
71
72
|
tags=[CryonirspTag.intermediate_frame(), CryonirspTag.task_bad_pixel_map()],
|
|
@@ -77,21 +78,19 @@ class BeamBoundariesCalibrationBase(CryonirspTaskBase):
|
|
|
77
78
|
)
|
|
78
79
|
|
|
79
80
|
# Step 3
|
|
80
|
-
with self.
|
|
81
|
+
with self.telemetry_span(f"Smooth the array to get good segmentation"):
|
|
81
82
|
smoothed_solar_gain_array = self.smooth_gain_array(corrected_solar_gain_array)
|
|
82
83
|
|
|
83
84
|
# Step 4
|
|
84
|
-
with self.
|
|
85
|
+
with self.telemetry_span(f"Split the beam horizontally"):
|
|
85
86
|
split_beams = self.split_beams(smoothed_solar_gain_array)
|
|
86
87
|
|
|
87
88
|
# Step 5
|
|
88
|
-
with self.
|
|
89
|
-
f"Segment the beams into illuminated and non-illuminated pixels"
|
|
90
|
-
):
|
|
89
|
+
with self.telemetry_span(f"Segment the beams into illuminated and non-illuminated pixels"):
|
|
91
90
|
segmented_beams = self.segment_arrays(split_beams)
|
|
92
91
|
|
|
93
92
|
# Step 6:
|
|
94
|
-
with self.
|
|
93
|
+
with self.telemetry_span(
|
|
95
94
|
f"Compute the inscribed rectangular extents of the illuminated portions of the sensor"
|
|
96
95
|
):
|
|
97
96
|
illuminated_boundaries = self.compute_boundaries_of_beam_illumination_regions(
|
|
@@ -99,14 +98,14 @@ class BeamBoundariesCalibrationBase(CryonirspTaskBase):
|
|
|
99
98
|
)
|
|
100
99
|
|
|
101
100
|
# Steps 7 - 9:
|
|
102
|
-
with self.
|
|
101
|
+
with self.telemetry_span(f"Compute the boundaries of the illuminated beams"):
|
|
103
102
|
split_beams_float = [split_beam.astype(float) for split_beam in split_beams]
|
|
104
103
|
boundaries = self.compute_final_beam_boundaries(
|
|
105
104
|
split_beams_float, illuminated_boundaries
|
|
106
105
|
)
|
|
107
106
|
|
|
108
107
|
# Step 10:
|
|
109
|
-
with self.
|
|
108
|
+
with self.telemetry_span("Writing beam boundaries"):
|
|
110
109
|
for beam, bounds in enumerate(boundaries, start=1):
|
|
111
110
|
self.write(
|
|
112
111
|
data=bounds.beam_boundaries_array,
|
|
@@ -233,7 +232,9 @@ class BeamBoundariesCalibrationBase(CryonirspTaskBase):
|
|
|
233
232
|
f"Unable to compute illuminated image boundaries for beam {beam}"
|
|
234
233
|
)
|
|
235
234
|
|
|
236
|
-
boundaries.append(
|
|
235
|
+
boundaries.append(
|
|
236
|
+
BeamBoundary(np.int64(y_min), np.int64(y_max), np.int64(x_min), np.int64(x_max))
|
|
237
|
+
)
|
|
237
238
|
|
|
238
239
|
return boundaries
|
|
239
240
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
"""CryoNIRSP base class."""
|
|
2
|
+
|
|
2
3
|
from abc import ABC
|
|
3
4
|
|
|
4
5
|
from dkist_processing_common.tasks import WorkflowTaskBase
|
|
5
|
-
from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetMixin
|
|
6
6
|
from dkist_processing_common.tasks.mixin.quality import QualityMixin
|
|
7
7
|
|
|
8
8
|
from dkist_processing_cryonirsp.models.constants import CryonirspConstants
|
|
@@ -12,7 +12,6 @@ from dkist_processing_cryonirsp.tasks.mixin.corrections import CorrectionsMixin
|
|
|
12
12
|
|
|
13
13
|
class CryonirspTaskBase(
|
|
14
14
|
WorkflowTaskBase,
|
|
15
|
-
InputDatasetMixin,
|
|
16
15
|
CorrectionsMixin,
|
|
17
16
|
QualityMixin,
|
|
18
17
|
ABC,
|
|
@@ -50,7 +49,7 @@ class CryonirspTaskBase(
|
|
|
50
49
|
workflow_version=workflow_version,
|
|
51
50
|
)
|
|
52
51
|
self.parameters = CryonirspParameters(
|
|
53
|
-
self.
|
|
52
|
+
scratch=self.scratch,
|
|
54
53
|
obs_ip_start_time=self.constants.obs_ip_start_time,
|
|
55
54
|
wavelength=self.constants.wavelength,
|
|
56
55
|
arm_id=self.constants.arm_id,
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""CryoNIRSP dark calibration task."""
|
|
2
|
+
|
|
2
3
|
from dkist_processing_common.codecs.fits import fits_array_encoder
|
|
3
4
|
from dkist_processing_common.models.task_name import TaskName
|
|
4
5
|
from dkist_processing_math.statistics import average_numpy_arrays
|
|
@@ -57,7 +58,7 @@ class DarkCalibration(CryonirspTaskBase):
|
|
|
57
58
|
)
|
|
58
59
|
|
|
59
60
|
logger.info(f"{target_exposure_conditions = }")
|
|
60
|
-
with self.
|
|
61
|
+
with self.telemetry_span(
|
|
61
62
|
f"Calculating dark frames for {self.constants.num_beams} beams and {len(target_exposure_conditions)} exp times"
|
|
62
63
|
):
|
|
63
64
|
total_dark_frames_used = 0
|
|
@@ -89,12 +90,12 @@ class DarkCalibration(CryonirspTaskBase):
|
|
|
89
90
|
beam_boundary=beam_boundary,
|
|
90
91
|
)
|
|
91
92
|
|
|
92
|
-
with self.
|
|
93
|
+
with self.telemetry_span(
|
|
93
94
|
f"Calculating dark for {exposure_conditions = } and {beam = }"
|
|
94
95
|
):
|
|
95
96
|
averaged_dark_array = average_numpy_arrays(linearized_dark_arrays)
|
|
96
97
|
|
|
97
|
-
with self.
|
|
98
|
+
with self.telemetry_span(
|
|
98
99
|
f"Writing dark for {exposure_conditions = } {beam = }"
|
|
99
100
|
):
|
|
100
101
|
self.write(
|
|
@@ -108,7 +109,7 @@ class DarkCalibration(CryonirspTaskBase):
|
|
|
108
109
|
encoder=fits_array_encoder,
|
|
109
110
|
)
|
|
110
111
|
|
|
111
|
-
with self.
|
|
112
|
+
with self.telemetry_span("Computing and logging quality metrics"):
|
|
112
113
|
no_of_raw_dark_frames: int = self.scratch.count_all(
|
|
113
114
|
tags=[
|
|
114
115
|
CryonirspTag.linearized_frame(),
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Cryo gain task."""
|
|
2
|
+
|
|
2
3
|
from abc import abstractmethod
|
|
3
4
|
|
|
4
5
|
import numpy as np
|
|
@@ -80,7 +81,7 @@ class GainCalibrationBase(CryonirspTaskBase):
|
|
|
80
81
|
target_exposure_conditions = self.exposure_conditions
|
|
81
82
|
|
|
82
83
|
logger.info(f"{target_exposure_conditions = }")
|
|
83
|
-
with self.
|
|
84
|
+
with self.telemetry_span(
|
|
84
85
|
f"Generate {self.gain_type} for {len(target_exposure_conditions)} exposure times"
|
|
85
86
|
):
|
|
86
87
|
for beam in range(1, self.constants.num_beams + 1):
|
|
@@ -95,7 +96,7 @@ class GainCalibrationBase(CryonirspTaskBase):
|
|
|
95
96
|
|
|
96
97
|
for exposure_conditions in target_exposure_conditions:
|
|
97
98
|
apm_str = f"{beam = } and {exposure_conditions = }"
|
|
98
|
-
with self.
|
|
99
|
+
with self.telemetry_span(f"Remove dark signal for {apm_str}"):
|
|
99
100
|
dark_array = next(
|
|
100
101
|
self.read(
|
|
101
102
|
tags=[
|
|
@@ -116,7 +117,7 @@ class GainCalibrationBase(CryonirspTaskBase):
|
|
|
116
117
|
subtract_array_from_arrays(avg_gain_array, dark_array)
|
|
117
118
|
)
|
|
118
119
|
|
|
119
|
-
with self.
|
|
120
|
+
with self.telemetry_span(f"Correct bad pixels for {apm_str}"):
|
|
120
121
|
bad_pixel_map = next(
|
|
121
122
|
self.read(
|
|
122
123
|
tags=[
|
|
@@ -132,12 +133,12 @@ class GainCalibrationBase(CryonirspTaskBase):
|
|
|
132
133
|
)
|
|
133
134
|
|
|
134
135
|
if self.normalize_gain_switch:
|
|
135
|
-
with self.
|
|
136
|
+
with self.telemetry_span(f"Normalize final gain for {apm_str}"):
|
|
136
137
|
normalized_gain_array = self.normalize_gain(bad_pixel_corrected_array)
|
|
137
138
|
else:
|
|
138
139
|
normalized_gain_array = bad_pixel_corrected_array
|
|
139
140
|
|
|
140
|
-
with self.
|
|
141
|
+
with self.telemetry_span(
|
|
141
142
|
f"Writing gain array for {beam = } and {exposure_conditions = }"
|
|
142
143
|
):
|
|
143
144
|
self.write(
|
|
@@ -151,7 +152,7 @@ class GainCalibrationBase(CryonirspTaskBase):
|
|
|
151
152
|
encoder=fits_array_encoder,
|
|
152
153
|
)
|
|
153
154
|
|
|
154
|
-
with self.
|
|
155
|
+
with self.telemetry_span("Computing and logging quality metrics"):
|
|
155
156
|
no_of_raw_gain_frames: int = self.scratch.count_all(
|
|
156
157
|
tags=[
|
|
157
158
|
CryonirspTag.linearized_frame(),
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Cryo instrument polarization task."""
|
|
2
|
+
|
|
2
3
|
from abc import ABC
|
|
3
4
|
from abc import abstractmethod
|
|
4
5
|
from collections import defaultdict
|
|
@@ -87,10 +88,10 @@ class InstrumentPolarizationCalibrationBase(CryonirspTaskBase, ABC):
|
|
|
87
88
|
f"Demodulation matrices will span FOV with shape {(self.parameters.polcal_num_spatial_bins, self.parameters.polcal_num_spatial_bins)}"
|
|
88
89
|
)
|
|
89
90
|
for beam in range(1, self.constants.num_beams + 1):
|
|
90
|
-
with self.
|
|
91
|
+
with self.telemetry_span(f"Reducing CS steps for {beam = }"):
|
|
91
92
|
local_reduced_arrays, global_reduced_arrays = self.reduce_cs_steps(beam)
|
|
92
93
|
|
|
93
|
-
with self.
|
|
94
|
+
with self.telemetry_span(f"Fit CU parameters for {beam = }"):
|
|
94
95
|
local_dresser = Dresser()
|
|
95
96
|
local_dresser.add_drawer(Drawer(local_reduced_arrays))
|
|
96
97
|
global_dresser = Dresser()
|
|
@@ -99,11 +100,11 @@ class InstrumentPolarizationCalibrationBase(CryonirspTaskBase, ABC):
|
|
|
99
100
|
local_dresser=local_dresser,
|
|
100
101
|
global_dresser=global_dresser,
|
|
101
102
|
fit_mode=self.parameters.polcal_pac_fit_mode,
|
|
102
|
-
init_set=self.
|
|
103
|
+
init_set=self.constants.pac_init_set,
|
|
103
104
|
fit_TM=False,
|
|
104
105
|
)
|
|
105
106
|
|
|
106
|
-
with self.
|
|
107
|
+
with self.telemetry_span(f"Resampling demodulation matrices for {beam = }"):
|
|
107
108
|
demod_matrices = pac_fitter.demodulation_matrices
|
|
108
109
|
# Reshaping the demodulation matrix to get rid of unit length dimensions
|
|
109
110
|
logger.info(f"Resampling demodulation matrices for {beam = }")
|
|
@@ -112,7 +113,7 @@ class InstrumentPolarizationCalibrationBase(CryonirspTaskBase, ABC):
|
|
|
112
113
|
f"Shape of resampled demodulation matrices for {beam = }: {demod_matrices.shape}"
|
|
113
114
|
)
|
|
114
115
|
|
|
115
|
-
with self.
|
|
116
|
+
with self.telemetry_span(f"Writing demodulation matrices for {beam = }"):
|
|
116
117
|
self.write(
|
|
117
118
|
data=demod_matrices,
|
|
118
119
|
tags=[
|
|
@@ -122,10 +123,10 @@ class InstrumentPolarizationCalibrationBase(CryonirspTaskBase, ABC):
|
|
|
122
123
|
encoder=fits_array_encoder,
|
|
123
124
|
)
|
|
124
125
|
|
|
125
|
-
with self.
|
|
126
|
+
with self.telemetry_span("Computing and recording polcal quality metrics"):
|
|
126
127
|
self.record_polcal_quality_metrics(beam, polcal_fitter=pac_fitter)
|
|
127
128
|
|
|
128
|
-
with self.
|
|
129
|
+
with self.telemetry_span("Computing and logging quality metrics"):
|
|
129
130
|
no_of_raw_polcal_frames: int = self.scratch.count_all(
|
|
130
131
|
tags=[
|
|
131
132
|
CryonirspTag.linearized_frame(),
|
|
@@ -270,11 +271,11 @@ class InstrumentPolarizationCalibrationBase(CryonirspTaskBase, ABC):
|
|
|
270
271
|
avg_inst_pol_cal_header = next(pol_cal_headers)
|
|
271
272
|
avg_inst_pol_cal_array = average_numpy_arrays(pol_cal_arrays)
|
|
272
273
|
|
|
273
|
-
with self.
|
|
274
|
+
with self.telemetry_span(f"Apply basic corrections for {apm_str}"):
|
|
274
275
|
dark_corrected_array = subtract_array_from_arrays(avg_inst_pol_cal_array, dark_array)
|
|
275
276
|
gain_corrected_array = next(divide_arrays_by_array(dark_corrected_array, gain_array))
|
|
276
277
|
|
|
277
|
-
with self.
|
|
278
|
+
with self.telemetry_span(f"Extract macro pixels from {apm_str}"):
|
|
278
279
|
self.set_original_beam_size(gain_corrected_array)
|
|
279
280
|
output_shape = (
|
|
280
281
|
self.parameters.polcal_num_spatial_bins,
|
|
@@ -283,7 +284,7 @@ class InstrumentPolarizationCalibrationBase(CryonirspTaskBase, ABC):
|
|
|
283
284
|
local_binned_array = next(resize_arrays(gain_corrected_array, output_shape))
|
|
284
285
|
global_binned_array = next(resize_arrays(gain_corrected_array, (1, 1)))
|
|
285
286
|
|
|
286
|
-
with self.
|
|
287
|
+
with self.telemetry_span(f"Create reduced CryonirspL0FitsAccess for {apm_str}"):
|
|
287
288
|
local_result = CryonirspL0FitsAccess(
|
|
288
289
|
fits.ImageHDU(local_binned_array[:, :], avg_inst_pol_cal_header),
|
|
289
290
|
auto_squeeze=False,
|
|
@@ -348,7 +349,7 @@ class InstrumentPolarizationCalibrationBase(CryonirspTaskBase, ABC):
|
|
|
348
349
|
self, target_exposure_conditions_list: [ExposureConditions]
|
|
349
350
|
):
|
|
350
351
|
"""Compute the polcal dark calibration."""
|
|
351
|
-
with self.
|
|
352
|
+
with self.telemetry_span(
|
|
352
353
|
f"Calculating dark frames for {len(target_exposure_conditions_list)} exp times"
|
|
353
354
|
):
|
|
354
355
|
for beam in range(1, self.constants.num_beams + 1):
|
|
@@ -360,7 +361,7 @@ class InstrumentPolarizationCalibrationBase(CryonirspTaskBase, ABC):
|
|
|
360
361
|
)
|
|
361
362
|
beam_boundary = BeamBoundary(*beam_array)
|
|
362
363
|
for exposure_conditions in target_exposure_conditions_list:
|
|
363
|
-
with self.
|
|
364
|
+
with self.telemetry_span(
|
|
364
365
|
f"Calculating polcal dark array(s) for {exposure_conditions = } and {beam = }"
|
|
365
366
|
):
|
|
366
367
|
linearized_dark_arrays = self.read(
|
|
@@ -375,7 +376,7 @@ class InstrumentPolarizationCalibrationBase(CryonirspTaskBase, ABC):
|
|
|
375
376
|
beam_boundary=beam_boundary,
|
|
376
377
|
)
|
|
377
378
|
averaged_dark_array = average_numpy_arrays(linearized_dark_arrays)
|
|
378
|
-
with self.
|
|
379
|
+
with self.telemetry_span(
|
|
379
380
|
f"Writing dark for {exposure_conditions = } and {beam = }"
|
|
380
381
|
):
|
|
381
382
|
self.write(
|
|
@@ -391,7 +392,7 @@ class InstrumentPolarizationCalibrationBase(CryonirspTaskBase, ABC):
|
|
|
391
392
|
|
|
392
393
|
def generate_polcal_gain_calibration(self, exposure_conditions_list: [ExposureConditions]):
|
|
393
394
|
"""Compute the polcal gain calibration."""
|
|
394
|
-
with self.
|
|
395
|
+
with self.telemetry_span(
|
|
395
396
|
f"Generate gains for {len(exposure_conditions_list)} exposure conditions"
|
|
396
397
|
):
|
|
397
398
|
for beam in range(1, self.constants.num_beams + 1):
|
|
@@ -422,7 +423,7 @@ class InstrumentPolarizationCalibrationBase(CryonirspTaskBase, ABC):
|
|
|
422
423
|
raise ValueError(
|
|
423
424
|
f"No matching polcal dark found for {exposure_conditions = } s and {beam = }"
|
|
424
425
|
) from e
|
|
425
|
-
with self.
|
|
426
|
+
with self.telemetry_span(
|
|
426
427
|
f"Calculating polcal gain array(s) for {exposure_conditions = } and {beam = }"
|
|
427
428
|
):
|
|
428
429
|
linearized_gain_arrays = self.read(
|
|
@@ -459,7 +460,7 @@ class InstrumentPolarizationCalibrationBase(CryonirspTaskBase, ABC):
|
|
|
459
460
|
bad_pixel_corrected_array
|
|
460
461
|
)
|
|
461
462
|
|
|
462
|
-
with self.
|
|
463
|
+
with self.telemetry_span(
|
|
463
464
|
f"Writing gain array for exposure time {exposure_conditions} and {beam = }"
|
|
464
465
|
):
|
|
465
466
|
self.write(
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Cryonirsp make movie frames task."""
|
|
2
|
+
|
|
2
3
|
from abc import ABC
|
|
3
4
|
from abc import abstractmethod
|
|
4
5
|
|
|
@@ -22,10 +23,10 @@ class MakeCryonirspMovieFramesBase(CryonirspTaskBase, ABC):
|
|
|
22
23
|
def run(self):
|
|
23
24
|
"""Create movie frames using all stokes states if they exist, otherwise only use intensity."""
|
|
24
25
|
if self.constants.correct_for_polarization:
|
|
25
|
-
with self.
|
|
26
|
+
with self.telemetry_span("Make full stokes movie"):
|
|
26
27
|
self.make_full_stokes_movie_frames()
|
|
27
28
|
else:
|
|
28
|
-
with self.
|
|
29
|
+
with self.telemetry_span("Make intensity only movie"):
|
|
29
30
|
self.make_intensity_movie_frames()
|
|
30
31
|
|
|
31
32
|
@abstractmethod
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Mixin to support array shift calculations."""
|
|
2
|
+
|
|
2
3
|
import math
|
|
3
4
|
from dataclasses import dataclass
|
|
4
5
|
|
|
@@ -123,10 +124,16 @@ class ShiftMeasurementsMixin:
|
|
|
123
124
|
array_2_strip, axis=array_2_axis_params.long_axis
|
|
124
125
|
)
|
|
125
126
|
|
|
127
|
+
if axis == SPECTRAL:
|
|
128
|
+
# Chop 10% off of the edges of the spectra to remove known edge effects before determining shift.
|
|
129
|
+
tenth_of_length = array_1_gradient.shape[0] // 10
|
|
130
|
+
array_1_gradient = array_1_gradient[tenth_of_length:-tenth_of_length]
|
|
131
|
+
array_2_gradient = array_2_gradient[tenth_of_length:-tenth_of_length]
|
|
132
|
+
|
|
126
133
|
# Correlate the gradient signals to get the shift of array 2 relative to array 1
|
|
127
134
|
shift_array, error, phasediff = skir.phase_cross_correlation(
|
|
128
|
-
array_1_gradient,
|
|
129
|
-
array_2_gradient,
|
|
135
|
+
reference_image=array_1_gradient,
|
|
136
|
+
moving_image=array_2_gradient,
|
|
130
137
|
upsample_factor=upsample_factor,
|
|
131
138
|
)
|
|
132
139
|
# Flip the sign to convert from correction to measurement.
|