dkist-processing-visp 2.20.14__py3-none-any.whl → 5.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dkist_processing_visp/__init__.py +1 -0
- dkist_processing_visp/config.py +1 -0
- dkist_processing_visp/models/constants.py +61 -20
- dkist_processing_visp/models/fits_access.py +20 -0
- dkist_processing_visp/models/metric_code.py +10 -0
- dkist_processing_visp/models/parameters.py +129 -24
- dkist_processing_visp/models/tags.py +22 -1
- dkist_processing_visp/models/task_name.py +1 -0
- dkist_processing_visp/parsers/map_repeats.py +1 -0
- dkist_processing_visp/parsers/modulator_states.py +1 -0
- dkist_processing_visp/parsers/polarimeter_mode.py +4 -2
- dkist_processing_visp/parsers/raster_step.py +4 -1
- dkist_processing_visp/parsers/spectrograph_configuration.py +75 -0
- dkist_processing_visp/parsers/time.py +24 -14
- dkist_processing_visp/parsers/visp_l0_fits_access.py +19 -8
- dkist_processing_visp/parsers/visp_l1_fits_access.py +1 -0
- dkist_processing_visp/tasks/__init__.py +1 -0
- dkist_processing_visp/tasks/assemble_movie.py +1 -0
- dkist_processing_visp/tasks/background_light.py +2 -1
- dkist_processing_visp/tasks/dark.py +5 -4
- dkist_processing_visp/tasks/geometric.py +132 -20
- dkist_processing_visp/tasks/instrument_polarization.py +128 -18
- dkist_processing_visp/tasks/l1_output_data.py +203 -0
- dkist_processing_visp/tasks/lamp.py +53 -93
- dkist_processing_visp/tasks/make_movie_frames.py +8 -6
- dkist_processing_visp/tasks/mixin/beam_access.py +1 -0
- dkist_processing_visp/tasks/mixin/corrections.py +54 -4
- dkist_processing_visp/tasks/mixin/downsample.py +1 -0
- dkist_processing_visp/tasks/parse.py +50 -17
- dkist_processing_visp/tasks/quality_metrics.py +5 -4
- dkist_processing_visp/tasks/science.py +126 -46
- dkist_processing_visp/tasks/solar.py +896 -456
- dkist_processing_visp/tasks/visp_base.py +4 -3
- dkist_processing_visp/tasks/write_l1.py +38 -10
- dkist_processing_visp/tests/conftest.py +145 -47
- dkist_processing_visp/tests/header_models.py +157 -20
- dkist_processing_visp/tests/local_trial_workflows/l0_cals_only.py +21 -78
- dkist_processing_visp/tests/local_trial_workflows/l0_polcals_as_science.py +421 -0
- dkist_processing_visp/tests/local_trial_workflows/l0_solar_gain_as_science.py +387 -0
- dkist_processing_visp/tests/local_trial_workflows/l0_to_l1.py +18 -75
- dkist_processing_visp/tests/local_trial_workflows/local_trial_helpers.py +346 -14
- dkist_processing_visp/tests/test_assemble_movie.py +2 -3
- dkist_processing_visp/tests/test_assemble_quality.py +89 -4
- dkist_processing_visp/tests/test_background_light.py +51 -44
- dkist_processing_visp/tests/test_dark.py +4 -3
- dkist_processing_visp/tests/test_downsample.py +1 -0
- dkist_processing_visp/tests/test_fits_access.py +43 -0
- dkist_processing_visp/tests/test_geometric.py +45 -4
- dkist_processing_visp/tests/test_instrument_polarization.py +72 -9
- dkist_processing_visp/tests/test_lamp.py +22 -26
- dkist_processing_visp/tests/test_make_movie_frames.py +4 -4
- dkist_processing_visp/tests/test_map_repeats.py +3 -1
- dkist_processing_visp/tests/test_parameters.py +122 -21
- dkist_processing_visp/tests/test_parse.py +164 -18
- dkist_processing_visp/tests/test_quality.py +3 -4
- dkist_processing_visp/tests/test_science.py +113 -15
- dkist_processing_visp/tests/test_solar.py +318 -99
- dkist_processing_visp/tests/test_visp_constants.py +38 -8
- dkist_processing_visp/tests/test_workflows.py +1 -0
- dkist_processing_visp/tests/test_write_l1.py +22 -3
- dkist_processing_visp/workflows/__init__.py +1 -0
- dkist_processing_visp/workflows/l0_processing.py +10 -3
- dkist_processing_visp/workflows/trial_workflows.py +8 -2
- dkist_processing_visp-5.1.1.dist-info/METADATA +552 -0
- dkist_processing_visp-5.1.1.dist-info/RECORD +94 -0
- {dkist_processing_visp-2.20.14.dist-info → dkist_processing_visp-5.1.1.dist-info}/WHEEL +1 -1
- docs/conf.py +5 -1
- docs/gain_correction.rst +52 -44
- docs/science_calibration.rst +7 -0
- dkist_processing_visp/tasks/mixin/line_zones.py +0 -115
- dkist_processing_visp-2.20.14.dist-info/METADATA +0 -196
- dkist_processing_visp-2.20.14.dist-info/RECORD +0 -89
- {dkist_processing_visp-2.20.14.dist-info → dkist_processing_visp-5.1.1.dist-info}/top_level.txt +0 -0
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
from dataclasses import asdict
|
|
2
2
|
from dataclasses import dataclass
|
|
3
3
|
|
|
4
|
+
import astropy.units as u
|
|
4
5
|
import numpy as np
|
|
5
6
|
import pytest
|
|
7
|
+
from hypothesis import HealthCheck
|
|
6
8
|
from hypothesis import example
|
|
7
9
|
from hypothesis import given
|
|
8
|
-
from hypothesis import HealthCheck
|
|
9
10
|
from hypothesis import settings
|
|
10
11
|
from hypothesis import strategies as st
|
|
12
|
+
from pydantic import BaseModel
|
|
11
13
|
|
|
12
14
|
from dkist_processing_visp.models.parameters import VispParameters
|
|
13
15
|
from dkist_processing_visp.models.parameters import VispParsingParameters
|
|
@@ -15,10 +17,27 @@ from dkist_processing_visp.tasks.visp_base import VispTaskBase
|
|
|
15
17
|
from dkist_processing_visp.tests.conftest import VispConstantsDb
|
|
16
18
|
from dkist_processing_visp.tests.conftest import VispInputDatasetParameterValues
|
|
17
19
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
|
|
21
|
+
@pytest.fixture(scope="session")
|
|
22
|
+
def parse_parameter_names() -> list[str]:
|
|
23
|
+
# The property names of all parameters on `VispParsingParameters`
|
|
24
|
+
return [k for k, v in vars(VispParsingParameters).items() if isinstance(v, property)]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@pytest.fixture(scope="session")
|
|
28
|
+
def arm_parameter_names() -> list[str]:
|
|
29
|
+
return [
|
|
30
|
+
"wavecal_camera_lens_parameters",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@pytest.fixture(scope="session")
|
|
35
|
+
def unit_parameter_names_and_units() -> dict[str, u.Unit | list[u.Unit]]:
|
|
36
|
+
return {
|
|
37
|
+
"solar_vignette_crval_bounds_px": u.pix,
|
|
38
|
+
"wavecal_camera_lens_parameters": [u.m, u.m / u.nm, u.m / u.nm**2],
|
|
39
|
+
"wavecal_pixel_pitch_micron_per_pix": u.um / u.pix,
|
|
40
|
+
}
|
|
22
41
|
|
|
23
42
|
|
|
24
43
|
@pytest.fixture(scope="function")
|
|
@@ -27,6 +46,7 @@ def basic_science_task_with_parameter_mixin(
|
|
|
27
46
|
assign_input_dataset_doc_to_task,
|
|
28
47
|
init_visp_constants_db,
|
|
29
48
|
testing_obs_ip_start_time,
|
|
49
|
+
arm_id,
|
|
30
50
|
):
|
|
31
51
|
def make_task(
|
|
32
52
|
parameters_part: dataclass,
|
|
@@ -34,8 +54,7 @@ def basic_science_task_with_parameter_mixin(
|
|
|
34
54
|
obs_ip_start_time=testing_obs_ip_start_time,
|
|
35
55
|
):
|
|
36
56
|
class Task(VispTaskBase):
|
|
37
|
-
def run(self):
|
|
38
|
-
...
|
|
57
|
+
def run(self): ...
|
|
39
58
|
|
|
40
59
|
init_visp_constants_db(recipe_run_id, VispConstantsDb())
|
|
41
60
|
task = Task(
|
|
@@ -49,8 +68,9 @@ def basic_science_task_with_parameter_mixin(
|
|
|
49
68
|
parameters_part,
|
|
50
69
|
parameter_class=parameter_class,
|
|
51
70
|
obs_ip_start_time=obs_ip_start_time,
|
|
71
|
+
arm_id=arm_id,
|
|
52
72
|
)
|
|
53
|
-
yield task, parameters_part
|
|
73
|
+
yield task, asdict(parameters_part)
|
|
54
74
|
except:
|
|
55
75
|
raise
|
|
56
76
|
finally:
|
|
@@ -59,7 +79,14 @@ def basic_science_task_with_parameter_mixin(
|
|
|
59
79
|
return make_task
|
|
60
80
|
|
|
61
81
|
|
|
62
|
-
|
|
82
|
+
@pytest.mark.parametrize("arm_id", [pytest.param("1"), pytest.param("2"), pytest.param("3")])
|
|
83
|
+
def test_non_wave_parameters(
|
|
84
|
+
basic_science_task_with_parameter_mixin,
|
|
85
|
+
parse_parameter_names,
|
|
86
|
+
arm_parameter_names,
|
|
87
|
+
unit_parameter_names_and_units,
|
|
88
|
+
arm_id,
|
|
89
|
+
):
|
|
63
90
|
"""
|
|
64
91
|
Given: A Science task with the parameter mixin
|
|
65
92
|
When: Accessing properties for parameters that do not depend on wavelength
|
|
@@ -69,15 +96,41 @@ def test_non_wave_parameters(basic_science_task_with_parameter_mixin):
|
|
|
69
96
|
basic_science_task_with_parameter_mixin(VispInputDatasetParameterValues())
|
|
70
97
|
)
|
|
71
98
|
task_param_attr = task.parameters
|
|
72
|
-
for
|
|
73
|
-
|
|
99
|
+
parameter_properties = [k for k, v in vars(VispParameters).items() if isinstance(v, property)]
|
|
100
|
+
for parameter_name in parameter_properties:
|
|
101
|
+
pn = f"visp_{parameter_name}"
|
|
102
|
+
if parameter_name in arm_parameter_names:
|
|
103
|
+
pn = f"{pn}_{arm_id}"
|
|
104
|
+
pv = expected[pn]
|
|
105
|
+
is_wavelength_param = isinstance(pv, dict) and "wavelength" in pv
|
|
74
106
|
if (
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
107
|
+
parameter_name not in parse_parameter_names
|
|
108
|
+
and not is_wavelength_param
|
|
109
|
+
and parameter_name != "solar_vignette_wavecal_fit_kwargs"
|
|
110
|
+
):
|
|
111
|
+
param_obj_value = getattr(task_param_attr, parameter_name)
|
|
112
|
+
if isinstance(pv, tuple):
|
|
113
|
+
pv = list(pv)
|
|
78
114
|
|
|
115
|
+
if parameter_name in unit_parameter_names_and_units:
|
|
116
|
+
expected_units = unit_parameter_names_and_units[parameter_name]
|
|
117
|
+
if not isinstance(param_obj_value, list):
|
|
118
|
+
param_obj_value = [param_obj_value]
|
|
119
|
+
pv = [pv]
|
|
120
|
+
expected_units = [expected_units]
|
|
79
121
|
|
|
80
|
-
|
|
122
|
+
assert all([param_obj_value[i].value == pv[i] for i in range(len(pv))])
|
|
123
|
+
assert all(
|
|
124
|
+
[param_obj_value[i].unit == expected_units[i] for i in range(len(pv))]
|
|
125
|
+
), f"Units {[v.unit for v in param_obj_value]} does not match expected {expected_units}"
|
|
126
|
+
elif isinstance(param_obj_value, BaseModel):
|
|
127
|
+
assert param_obj_value.model_dump() == pv
|
|
128
|
+
else:
|
|
129
|
+
assert getattr(task_param_attr, parameter_name) == pv
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@pytest.mark.parametrize("arm_id", ["1"])
|
|
133
|
+
def test_parse_parameters(basic_science_task_with_parameter_mixin, parse_parameter_names):
|
|
81
134
|
"""
|
|
82
135
|
Given: A Science task with Parsing parameters
|
|
83
136
|
When: Accessing properties for Parse parameters
|
|
@@ -91,16 +144,19 @@ def test_parse_parameters(basic_science_task_with_parameter_mixin):
|
|
|
91
144
|
)
|
|
92
145
|
)
|
|
93
146
|
task_param_attr = task.parameters
|
|
94
|
-
for pn, pv in
|
|
147
|
+
for pn, pv in expected.items():
|
|
95
148
|
property_name = pn.removeprefix("visp_")
|
|
96
|
-
if property_name in
|
|
149
|
+
if property_name in parse_parameter_names and type(pv) is not dict:
|
|
97
150
|
assert getattr(task_param_attr, property_name) == pv
|
|
98
151
|
|
|
99
152
|
|
|
100
153
|
@given(wave=st.floats(min_value=500.0, max_value=2000.0))
|
|
101
154
|
@settings(suppress_health_check=[HealthCheck.function_scoped_fixture])
|
|
102
155
|
@example(wave=492.5)
|
|
103
|
-
|
|
156
|
+
@pytest.mark.parametrize("arm_id", ["1"])
|
|
157
|
+
def test_wave_parameters(
|
|
158
|
+
basic_science_task_with_parameter_mixin, parse_parameter_names, arm_parameter_names, wave
|
|
159
|
+
):
|
|
104
160
|
"""
|
|
105
161
|
Given: A Science task with the paramter mixin
|
|
106
162
|
When: Accessing properties for parameters that depend on wavelength
|
|
@@ -111,10 +167,55 @@ def test_wave_parameters(basic_science_task_with_parameter_mixin, wave):
|
|
|
111
167
|
)
|
|
112
168
|
task_param_attr = task.parameters
|
|
113
169
|
task_param_attr._wavelength = wave
|
|
114
|
-
pwaves = np.array(expected
|
|
170
|
+
pwaves = np.array(expected["visp_geo_zone_normalization_percentile"]["wavelength"])
|
|
115
171
|
midpoints = 0.5 * (pwaves[1:] + pwaves[:-1])
|
|
116
172
|
idx = np.sum(midpoints < wave)
|
|
117
|
-
for pn, pv in
|
|
173
|
+
for pn, pv in expected.items():
|
|
118
174
|
property_name = pn.removeprefix("visp_")
|
|
119
|
-
|
|
175
|
+
is_wavelength_param = isinstance(pv, dict) and "wavelength" in pv
|
|
176
|
+
if is_wavelength_param and property_name not in parse_parameter_names + arm_parameter_names:
|
|
120
177
|
assert getattr(task_param_attr, property_name) == pv["values"][idx]
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
class AnyInt:
|
|
181
|
+
pass
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@pytest.mark.parametrize("arm_id", [pytest.param("1")])
|
|
185
|
+
@pytest.mark.parametrize(
|
|
186
|
+
"db_value, expected",
|
|
187
|
+
[
|
|
188
|
+
pytest.param({"method": "nelder"}, {"method": "nelder"}, id="non_rng_method"),
|
|
189
|
+
pytest.param(
|
|
190
|
+
{"method": "basinhopping"}, {"method": "basinhopping", "rng": AnyInt}, id="random_rng"
|
|
191
|
+
),
|
|
192
|
+
pytest.param(
|
|
193
|
+
{"method": "differential_evolution", "rng": 6.28},
|
|
194
|
+
{"method": "differential_evolution", "rng": 6.28},
|
|
195
|
+
id="override_rng",
|
|
196
|
+
),
|
|
197
|
+
pytest.param(dict(), dict(), id="no_kwargs"),
|
|
198
|
+
],
|
|
199
|
+
)
|
|
200
|
+
def test_fit_kwarg_parameters(
|
|
201
|
+
basic_science_task_with_parameter_mixin,
|
|
202
|
+
db_value,
|
|
203
|
+
expected,
|
|
204
|
+
):
|
|
205
|
+
"""
|
|
206
|
+
Given: A Science task with the parameter mixin
|
|
207
|
+
When: Accessing properties for parameters that do not depend on wavelength
|
|
208
|
+
Then: The correct value is returned
|
|
209
|
+
"""
|
|
210
|
+
task, _ = next(
|
|
211
|
+
basic_science_task_with_parameter_mixin(
|
|
212
|
+
VispInputDatasetParameterValues(visp_solar_vignette_wavecal_fit_kwargs=db_value)
|
|
213
|
+
)
|
|
214
|
+
)
|
|
215
|
+
kwarg_dict = task.parameters.solar_vignette_wavecal_fit_kwargs
|
|
216
|
+
assert kwarg_dict.keys() == expected.keys()
|
|
217
|
+
for k in expected.keys():
|
|
218
|
+
if expected[k] is AnyInt:
|
|
219
|
+
assert type(kwarg_dict[k]) is int
|
|
220
|
+
else:
|
|
221
|
+
assert expected[k] == kwarg_dict[k]
|
|
@@ -2,7 +2,6 @@ import pytest
|
|
|
2
2
|
from dkist_data_simulator.dataset import key_function
|
|
3
3
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
4
4
|
from dkist_processing_common.models.tags import Tag
|
|
5
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
6
5
|
|
|
7
6
|
from dkist_processing_visp.models.parameters import VispParsingParameters
|
|
8
7
|
from dkist_processing_visp.models.tags import VispTag
|
|
@@ -11,7 +10,9 @@ from dkist_processing_visp.tests.conftest import VispInputDatasetParameterValues
|
|
|
11
10
|
from dkist_processing_visp.tests.conftest import write_frames_to_task
|
|
12
11
|
from dkist_processing_visp.tests.header_models import VispHeadersInputDarkFrames
|
|
13
12
|
from dkist_processing_visp.tests.header_models import VispHeadersInputLampGainFrames
|
|
13
|
+
from dkist_processing_visp.tests.header_models import VispHeadersInputPolcalDarkFrames
|
|
14
14
|
from dkist_processing_visp.tests.header_models import VispHeadersInputPolcalFrames
|
|
15
|
+
from dkist_processing_visp.tests.header_models import VispHeadersInputPolcalGainFrames
|
|
15
16
|
from dkist_processing_visp.tests.header_models import VispHeadersInputSolarGainFrames
|
|
16
17
|
from dkist_processing_visp.tests.header_models import VispHeadersValidObserveFrames
|
|
17
18
|
|
|
@@ -40,13 +41,11 @@ def observe_readout_exp_times() -> list[float]:
|
|
|
40
41
|
def required_dark_readout_exp_times(
|
|
41
42
|
lamp_readout_exp_time,
|
|
42
43
|
solar_readout_exp_time,
|
|
43
|
-
polcal_readout_exp_time,
|
|
44
44
|
observe_readout_exp_times,
|
|
45
45
|
) -> list[float]:
|
|
46
46
|
return [
|
|
47
47
|
lamp_readout_exp_time,
|
|
48
48
|
solar_readout_exp_time,
|
|
49
|
-
polcal_readout_exp_time,
|
|
50
49
|
] + observe_readout_exp_times
|
|
51
50
|
|
|
52
51
|
|
|
@@ -87,6 +86,7 @@ def write_input_dark_frames_to_task(
|
|
|
87
86
|
time_delta: float = 10.0,
|
|
88
87
|
num_modstates: int = 2,
|
|
89
88
|
data_shape: tuple[int, int] = (2, 2),
|
|
89
|
+
**kwargs,
|
|
90
90
|
):
|
|
91
91
|
array_shape = (1, *data_shape)
|
|
92
92
|
dataset = VispHeadersInputDarkFrames(
|
|
@@ -95,6 +95,7 @@ def write_input_dark_frames_to_task(
|
|
|
95
95
|
exp_time=exp_time,
|
|
96
96
|
readout_exp_time=readout_exp_time,
|
|
97
97
|
num_modstates=num_modstates,
|
|
98
|
+
**kwargs,
|
|
98
99
|
)
|
|
99
100
|
|
|
100
101
|
num_written_frames = write_frames_to_task(
|
|
@@ -110,6 +111,7 @@ def write_input_lamp_frames_to_task(
|
|
|
110
111
|
time_delta: float = 10.0,
|
|
111
112
|
num_modstates: int = 2,
|
|
112
113
|
data_shape: tuple[int, int] = (2, 2),
|
|
114
|
+
**kwargs,
|
|
113
115
|
):
|
|
114
116
|
array_shape = (1, *data_shape)
|
|
115
117
|
dataset = VispHeadersInputLampGainFrames(
|
|
@@ -118,6 +120,7 @@ def write_input_lamp_frames_to_task(
|
|
|
118
120
|
exp_time=exp_time,
|
|
119
121
|
readout_exp_time=readout_exp_time,
|
|
120
122
|
num_modstates=num_modstates,
|
|
123
|
+
**kwargs,
|
|
121
124
|
)
|
|
122
125
|
|
|
123
126
|
num_written_frames = write_frames_to_task(
|
|
@@ -133,6 +136,7 @@ def write_input_solar_frames_to_task(
|
|
|
133
136
|
time_delta: float = 10.0,
|
|
134
137
|
num_modstates: int = 2,
|
|
135
138
|
data_shape: tuple[int, int] = (2, 2),
|
|
139
|
+
**kwargs,
|
|
136
140
|
):
|
|
137
141
|
array_shape = (1, *data_shape)
|
|
138
142
|
dataset = VispHeadersInputSolarGainFrames(
|
|
@@ -141,6 +145,7 @@ def write_input_solar_frames_to_task(
|
|
|
141
145
|
exp_time=exp_time,
|
|
142
146
|
readout_exp_time=readout_exp_time,
|
|
143
147
|
num_modstates=num_modstates,
|
|
148
|
+
**kwargs,
|
|
144
149
|
)
|
|
145
150
|
|
|
146
151
|
num_written_frames = write_frames_to_task(
|
|
@@ -156,6 +161,7 @@ def write_input_polcal_frames_to_task(
|
|
|
156
161
|
time_delta: float = 30.0,
|
|
157
162
|
num_modstates: int = 2,
|
|
158
163
|
data_shape: tuple[int, int] = (2, 2),
|
|
164
|
+
**kwargs,
|
|
159
165
|
):
|
|
160
166
|
array_shape = (1, *data_shape)
|
|
161
167
|
dataset = VispHeadersInputPolcalFrames(
|
|
@@ -164,6 +170,57 @@ def write_input_polcal_frames_to_task(
|
|
|
164
170
|
exp_time=exp_time,
|
|
165
171
|
readout_exp_time=readout_exp_time,
|
|
166
172
|
num_modstates=num_modstates,
|
|
173
|
+
**kwargs,
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
num_written_frames = write_frames_to_task(
|
|
177
|
+
task=task, frame_generator=dataset, extra_tags=[VispTag.input()]
|
|
178
|
+
)
|
|
179
|
+
return num_written_frames
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def write_input_polcal_dark_frames_to_task(
|
|
183
|
+
task,
|
|
184
|
+
readout_exp_time: float,
|
|
185
|
+
exp_time: float,
|
|
186
|
+
time_delta: float = 30.0,
|
|
187
|
+
num_modstates: int = 2,
|
|
188
|
+
data_shape: tuple[int, int] = (2, 2),
|
|
189
|
+
**kwargs,
|
|
190
|
+
):
|
|
191
|
+
array_shape = (1, *data_shape)
|
|
192
|
+
dataset = VispHeadersInputPolcalDarkFrames(
|
|
193
|
+
array_shape=array_shape,
|
|
194
|
+
time_delta=time_delta,
|
|
195
|
+
exp_time=exp_time,
|
|
196
|
+
readout_exp_time=readout_exp_time,
|
|
197
|
+
num_modstates=num_modstates,
|
|
198
|
+
**kwargs,
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
num_written_frames = write_frames_to_task(
|
|
202
|
+
task=task, frame_generator=dataset, extra_tags=[VispTag.input()]
|
|
203
|
+
)
|
|
204
|
+
return num_written_frames
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def write_input_polcal_gain_frames_to_task(
|
|
208
|
+
task,
|
|
209
|
+
readout_exp_time: float,
|
|
210
|
+
exp_time: float,
|
|
211
|
+
time_delta: float = 30.0,
|
|
212
|
+
num_modstates: int = 2,
|
|
213
|
+
data_shape: tuple[int, int] = (2, 2),
|
|
214
|
+
**kwargs,
|
|
215
|
+
):
|
|
216
|
+
array_shape = (1, *data_shape)
|
|
217
|
+
dataset = VispHeadersInputPolcalGainFrames(
|
|
218
|
+
array_shape=array_shape,
|
|
219
|
+
time_delta=time_delta,
|
|
220
|
+
exp_time=exp_time,
|
|
221
|
+
readout_exp_time=readout_exp_time,
|
|
222
|
+
num_modstates=num_modstates,
|
|
223
|
+
**kwargs,
|
|
167
224
|
)
|
|
168
225
|
|
|
169
226
|
num_written_frames = write_frames_to_task(
|
|
@@ -182,6 +239,7 @@ def write_input_observe_frames_to_task(
|
|
|
182
239
|
time_delta: float = 10.0,
|
|
183
240
|
data_shape: tuple[int, int] = (2, 2),
|
|
184
241
|
obs_dataset_class=VispHeadersValidObserveFrames,
|
|
242
|
+
**kwargs,
|
|
185
243
|
):
|
|
186
244
|
array_shape = (1, *data_shape)
|
|
187
245
|
dataset = obs_dataset_class(
|
|
@@ -192,6 +250,7 @@ def write_input_observe_frames_to_task(
|
|
|
192
250
|
num_modstates=num_modstates,
|
|
193
251
|
exp_time=exp_time,
|
|
194
252
|
readout_exp_time=readout_exp_time,
|
|
253
|
+
**kwargs,
|
|
195
254
|
)
|
|
196
255
|
num_written_frames = write_frames_to_task(
|
|
197
256
|
task=task, frame_generator=dataset, extra_tags=[VispTag.input()]
|
|
@@ -227,6 +286,11 @@ def write_input_cal_frames_to_task(
|
|
|
227
286
|
solar_exp_time,
|
|
228
287
|
polcal_exp_time,
|
|
229
288
|
num_modstates,
|
|
289
|
+
testing_arm_id,
|
|
290
|
+
testing_solar_ip_start_time,
|
|
291
|
+
testing_grating_constant,
|
|
292
|
+
testing_grating_angle,
|
|
293
|
+
testing_arm_position,
|
|
230
294
|
):
|
|
231
295
|
def write_frames_to_task(task):
|
|
232
296
|
for readout_exp_time in required_dark_readout_exp_times:
|
|
@@ -235,6 +299,7 @@ def write_input_cal_frames_to_task(
|
|
|
235
299
|
readout_exp_time=readout_exp_time,
|
|
236
300
|
exp_time=dark_exp_time,
|
|
237
301
|
num_modstates=num_modstates,
|
|
302
|
+
arm_id=testing_arm_id,
|
|
238
303
|
)
|
|
239
304
|
|
|
240
305
|
write_input_lamp_frames_to_task(
|
|
@@ -242,18 +307,39 @@ def write_input_cal_frames_to_task(
|
|
|
242
307
|
readout_exp_time=lamp_readout_exp_time,
|
|
243
308
|
exp_time=lamp_exp_time,
|
|
244
309
|
num_modstates=num_modstates,
|
|
310
|
+
arm_id=testing_arm_id,
|
|
245
311
|
)
|
|
246
312
|
write_input_solar_frames_to_task(
|
|
247
313
|
task=task,
|
|
248
314
|
readout_exp_time=solar_readout_exp_time,
|
|
249
315
|
exp_time=solar_exp_time,
|
|
250
316
|
num_modstates=num_modstates,
|
|
317
|
+
arm_id=testing_arm_id,
|
|
318
|
+
ip_start_time=testing_solar_ip_start_time,
|
|
319
|
+
grating_constant=testing_grating_constant,
|
|
320
|
+
grating_angle=testing_grating_angle,
|
|
321
|
+
arm_position=testing_arm_position,
|
|
251
322
|
)
|
|
252
323
|
write_input_polcal_frames_to_task(
|
|
253
324
|
task=task,
|
|
254
325
|
readout_exp_time=polcal_readout_exp_time,
|
|
255
326
|
exp_time=polcal_exp_time,
|
|
256
327
|
num_modstates=num_modstates,
|
|
328
|
+
arm_id=testing_arm_id,
|
|
329
|
+
)
|
|
330
|
+
write_input_polcal_dark_frames_to_task(
|
|
331
|
+
task=task,
|
|
332
|
+
readout_exp_time=polcal_readout_exp_time,
|
|
333
|
+
exp_time=polcal_exp_time,
|
|
334
|
+
num_modstates=num_modstates,
|
|
335
|
+
arm_id=testing_arm_id,
|
|
336
|
+
)
|
|
337
|
+
write_input_polcal_gain_frames_to_task(
|
|
338
|
+
task,
|
|
339
|
+
readout_exp_time=polcal_readout_exp_time,
|
|
340
|
+
exp_time=polcal_exp_time,
|
|
341
|
+
num_modstates=num_modstates,
|
|
342
|
+
arm_id=testing_arm_id,
|
|
257
343
|
)
|
|
258
344
|
|
|
259
345
|
return write_frames_to_task
|
|
@@ -292,6 +378,11 @@ def test_parse_visp_input_data(
|
|
|
292
378
|
observe_exp_times,
|
|
293
379
|
num_modstates,
|
|
294
380
|
mocker,
|
|
381
|
+
fake_gql_client,
|
|
382
|
+
testing_arm_id,
|
|
383
|
+
testing_grating_constant,
|
|
384
|
+
testing_grating_angle,
|
|
385
|
+
testing_arm_position,
|
|
295
386
|
):
|
|
296
387
|
"""
|
|
297
388
|
Given: A ParseVispInputData task
|
|
@@ -299,7 +390,7 @@ def test_parse_visp_input_data(
|
|
|
299
390
|
Then: All tagged files exist and individual task tags are applied
|
|
300
391
|
"""
|
|
301
392
|
mocker.patch(
|
|
302
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
393
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
303
394
|
)
|
|
304
395
|
task = parse_task_with_no_data
|
|
305
396
|
write_input_cal_frames_to_task(task)
|
|
@@ -311,6 +402,10 @@ def test_parse_visp_input_data(
|
|
|
311
402
|
num_steps=3,
|
|
312
403
|
readout_exp_time=obs_readout_exp_time,
|
|
313
404
|
exp_time=obs_exp_time,
|
|
405
|
+
arm_id=testing_arm_id,
|
|
406
|
+
grating_constant=testing_grating_constant,
|
|
407
|
+
grating_angle=testing_grating_angle,
|
|
408
|
+
arm_position=testing_arm_position,
|
|
314
409
|
)
|
|
315
410
|
|
|
316
411
|
# When
|
|
@@ -323,14 +418,19 @@ def test_parse_visp_input_data(
|
|
|
323
418
|
assert list(task.read(tags=[Tag.input(), Tag.task_dark()]))
|
|
324
419
|
assert list(task.read(tags=[Tag.input(), Tag.task_lamp_gain()]))
|
|
325
420
|
assert list(task.read(tags=[Tag.input(), Tag.task_solar_gain()]))
|
|
326
|
-
assert
|
|
421
|
+
assert (
|
|
422
|
+
len(list(task.read(tags=[Tag.input(), Tag.task_polcal()]))) == 6
|
|
423
|
+
) # 2 polcal observes, 2 darks, 2 gains
|
|
327
424
|
assert list(task.read(tags=[Tag.input(), Tag.task_observe()]))
|
|
425
|
+
assert len(list(task.read(tags=[Tag.input(), Tag.task_polcal_dark()]))) == 2 # 2 polcal darks
|
|
426
|
+
assert len(list(task.read(tags=[Tag.input(), Tag.task_polcal_gain()]))) == 2 # 2 polcal gains
|
|
328
427
|
|
|
329
428
|
|
|
330
429
|
def test_parse_visp_input_data_constants(
|
|
331
430
|
parse_task_with_no_data,
|
|
332
431
|
write_input_cal_frames_to_task,
|
|
333
432
|
mocker,
|
|
433
|
+
fake_gql_client,
|
|
334
434
|
lamp_readout_exp_time,
|
|
335
435
|
solar_readout_exp_time,
|
|
336
436
|
polcal_readout_exp_time,
|
|
@@ -341,6 +441,12 @@ def test_parse_visp_input_data_constants(
|
|
|
341
441
|
polcal_exp_time,
|
|
342
442
|
observe_exp_times,
|
|
343
443
|
num_modstates,
|
|
444
|
+
testing_arm_id,
|
|
445
|
+
testing_obs_ip_start_time,
|
|
446
|
+
testing_solar_ip_start_time,
|
|
447
|
+
testing_grating_constant,
|
|
448
|
+
testing_grating_angle,
|
|
449
|
+
testing_arm_position,
|
|
344
450
|
):
|
|
345
451
|
"""
|
|
346
452
|
Given: A ParseVispInputData task
|
|
@@ -348,7 +454,7 @@ def test_parse_visp_input_data_constants(
|
|
|
348
454
|
Then: Constants are in the constants object as expected
|
|
349
455
|
"""
|
|
350
456
|
mocker.patch(
|
|
351
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
457
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
352
458
|
)
|
|
353
459
|
task = parse_task_with_no_data
|
|
354
460
|
write_input_cal_frames_to_task(task)
|
|
@@ -363,17 +469,29 @@ def test_parse_visp_input_data_constants(
|
|
|
363
469
|
num_steps=num_steps,
|
|
364
470
|
readout_exp_time=obs_readout_exp_time,
|
|
365
471
|
exp_time=obs_exp_time,
|
|
472
|
+
arm_id=testing_arm_id,
|
|
473
|
+
ip_start_time=testing_obs_ip_start_time,
|
|
474
|
+
grating_constant=testing_grating_constant,
|
|
475
|
+
grating_angle=testing_grating_angle,
|
|
476
|
+
arm_position=testing_arm_position,
|
|
366
477
|
)
|
|
367
478
|
|
|
368
479
|
# When
|
|
369
480
|
task()
|
|
370
481
|
# Then
|
|
482
|
+
assert task.constants._db_dict["ARM_ID"] == testing_arm_id
|
|
371
483
|
expected_dark_readout_exp_times = [
|
|
372
484
|
lamp_readout_exp_time,
|
|
373
485
|
solar_readout_exp_time,
|
|
374
|
-
polcal_readout_exp_time,
|
|
375
486
|
] + observe_readout_exp_times
|
|
376
|
-
assert task.constants._db_dict["OBS_IP_START_TIME"] ==
|
|
487
|
+
assert task.constants._db_dict["OBS_IP_START_TIME"] == testing_obs_ip_start_time
|
|
488
|
+
assert task.constants._db_dict["INCIDENT_LIGHT_ANGLE_DEG"] == -1 * testing_grating_angle
|
|
489
|
+
assert (
|
|
490
|
+
task.constants._db_dict["REFLECTED_LIGHT_ANGLE_DEG"]
|
|
491
|
+
== -1 * testing_grating_angle + testing_arm_position
|
|
492
|
+
)
|
|
493
|
+
assert task.constants._db_dict["GRATING_CONSTANT_INVERSE_MM"] == testing_grating_constant
|
|
494
|
+
assert task.constants._db_dict["SOLAR_GAIN_IP_START_TIME"] == testing_solar_ip_start_time
|
|
377
495
|
assert task.constants._db_dict["NUM_MODSTATES"] == num_modstates
|
|
378
496
|
assert task.constants._db_dict["NUM_MAP_SCANS"] == num_maps_per_readout_exp_time * len(
|
|
379
497
|
observe_readout_exp_times
|
|
@@ -392,6 +510,11 @@ def test_parse_visp_input_data_constants(
|
|
|
392
510
|
assert sorted(task.constants._db_dict["OBSERVE_READOUT_EXP_TIMES"]) == sorted(
|
|
393
511
|
observe_readout_exp_times
|
|
394
512
|
)
|
|
513
|
+
assert task.constants._db_dict["RETARDER_NAME"] == "SiO2 OC"
|
|
514
|
+
assert task.constants._db_dict["DARK_GOS_LEVEL3_STATUS"] == "lamp"
|
|
515
|
+
assert task.constants._db_dict["SOLAR_GAIN_GOS_LEVEL3_STATUS"] == "clear"
|
|
516
|
+
assert task.constants._db_dict["SOLAR_GAIN_NUM_RAW_FRAMES_PER_FPA"] == 10
|
|
517
|
+
assert task.constants._db_dict["POLCAL_NUM_RAW_FRAMES_PER_FPA"] == 10
|
|
395
518
|
|
|
396
519
|
|
|
397
520
|
def test_parse_visp_values(
|
|
@@ -400,6 +523,11 @@ def test_parse_visp_values(
|
|
|
400
523
|
observe_readout_exp_times,
|
|
401
524
|
num_modstates,
|
|
402
525
|
mocker,
|
|
526
|
+
fake_gql_client,
|
|
527
|
+
testing_arm_id,
|
|
528
|
+
testing_grating_constant,
|
|
529
|
+
testing_grating_angle,
|
|
530
|
+
testing_arm_position,
|
|
403
531
|
):
|
|
404
532
|
"""
|
|
405
533
|
:Given: A valid parse input task
|
|
@@ -407,7 +535,7 @@ def test_parse_visp_values(
|
|
|
407
535
|
:Then: Values are correctly loaded into the constants mutable mapping
|
|
408
536
|
"""
|
|
409
537
|
mocker.patch(
|
|
410
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
538
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
411
539
|
)
|
|
412
540
|
task = parse_task_with_no_data
|
|
413
541
|
write_input_cal_frames_to_task(task)
|
|
@@ -418,6 +546,10 @@ def test_parse_visp_values(
|
|
|
418
546
|
num_maps=1,
|
|
419
547
|
num_steps=1,
|
|
420
548
|
num_modstates=num_modstates,
|
|
549
|
+
arm_id=testing_arm_id,
|
|
550
|
+
grating_constant=testing_grating_constant,
|
|
551
|
+
grating_angle=testing_grating_angle,
|
|
552
|
+
arm_position=testing_arm_position,
|
|
421
553
|
)
|
|
422
554
|
|
|
423
555
|
task()
|
|
@@ -426,16 +558,19 @@ def test_parse_visp_values(
|
|
|
426
558
|
assert task.constants.maximum_cadence == 10
|
|
427
559
|
assert task.constants.minimum_cadence == 10
|
|
428
560
|
assert task.constants.variance_cadence == 0
|
|
561
|
+
assert task.constants.camera_name == "camera_name"
|
|
429
562
|
|
|
430
563
|
|
|
431
|
-
def test_multiple_num_raster_steps_raises_error(
|
|
564
|
+
def test_multiple_num_raster_steps_raises_error(
|
|
565
|
+
parse_task_with_no_data, num_modstates, mocker, fake_gql_client
|
|
566
|
+
):
|
|
432
567
|
"""
|
|
433
568
|
:Given: A prase task with data that have inconsistent VSPNSTP values
|
|
434
569
|
:When: Calling the parse task
|
|
435
570
|
:Then: The correct error is raised
|
|
436
571
|
"""
|
|
437
572
|
mocker.patch(
|
|
438
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
573
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
439
574
|
)
|
|
440
575
|
task = parse_task_with_no_data
|
|
441
576
|
write_input_dark_frames_to_task(task, readout_exp_time=0.1, exp_time=0.2)
|
|
@@ -453,14 +588,14 @@ def test_multiple_num_raster_steps_raises_error(parse_task_with_no_data, num_mod
|
|
|
453
588
|
task()
|
|
454
589
|
|
|
455
590
|
|
|
456
|
-
def test_incomplete_single_map(parse_task_with_no_data, num_modstates, mocker):
|
|
591
|
+
def test_incomplete_single_map(parse_task_with_no_data, num_modstates, mocker, fake_gql_client):
|
|
457
592
|
"""
|
|
458
593
|
:Given: A parse task with data that has an incomplete raster scan
|
|
459
594
|
:When: Calling the parse task
|
|
460
595
|
:Then: The correct number of raster steps are found
|
|
461
596
|
"""
|
|
462
597
|
mocker.patch(
|
|
463
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
598
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
464
599
|
)
|
|
465
600
|
task = parse_task_with_no_data
|
|
466
601
|
num_steps = 4
|
|
@@ -480,14 +615,14 @@ def test_incomplete_single_map(parse_task_with_no_data, num_modstates, mocker):
|
|
|
480
615
|
assert task.constants._db_dict["NUM_MAP_SCANS"] == num_map_scans
|
|
481
616
|
|
|
482
617
|
|
|
483
|
-
def test_incomplete_final_map(parse_task_with_no_data, num_modstates, mocker):
|
|
618
|
+
def test_incomplete_final_map(parse_task_with_no_data, num_modstates, mocker, fake_gql_client):
|
|
484
619
|
"""
|
|
485
620
|
:Given: A parse task with data that has complete raster scans along with an incomplete raster scan
|
|
486
621
|
:When: Calling the parse task
|
|
487
622
|
:Then: The correct number of raster steps and maps are found
|
|
488
623
|
"""
|
|
489
624
|
mocker.patch(
|
|
490
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
625
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
491
626
|
)
|
|
492
627
|
task = parse_task_with_no_data
|
|
493
628
|
num_steps = 4
|
|
@@ -513,6 +648,11 @@ def test_intensity_observes_and_polarimetric_cals(
|
|
|
513
648
|
observe_readout_exp_times,
|
|
514
649
|
observe_exp_times,
|
|
515
650
|
mocker,
|
|
651
|
+
fake_gql_client,
|
|
652
|
+
testing_arm_id,
|
|
653
|
+
testing_grating_constant,
|
|
654
|
+
testing_grating_angle,
|
|
655
|
+
testing_arm_position,
|
|
516
656
|
):
|
|
517
657
|
"""
|
|
518
658
|
:Given: Data where the observe frames are in intensity mode and the calibration frames are in polarimetric mode
|
|
@@ -520,7 +660,7 @@ def test_intensity_observes_and_polarimetric_cals(
|
|
|
520
660
|
:Then: All modulator state keys generated for all frames are in the first modulator state
|
|
521
661
|
"""
|
|
522
662
|
mocker.patch(
|
|
523
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
663
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
524
664
|
)
|
|
525
665
|
task = parse_task_with_no_data
|
|
526
666
|
write_input_cal_frames_to_task(task)
|
|
@@ -532,6 +672,10 @@ def test_intensity_observes_and_polarimetric_cals(
|
|
|
532
672
|
readout_exp_time=observe_readout_exp_times[0],
|
|
533
673
|
exp_time=observe_exp_times[0],
|
|
534
674
|
obs_dataset_class=VispHeadersIntensityObserveFrames,
|
|
675
|
+
arm_id=testing_arm_id,
|
|
676
|
+
grating_constant=testing_grating_constant,
|
|
677
|
+
grating_angle=testing_grating_angle,
|
|
678
|
+
arm_position=testing_arm_position,
|
|
535
679
|
)
|
|
536
680
|
task()
|
|
537
681
|
assert task.constants._db_dict["NUM_MODSTATES"] == 1
|
|
@@ -541,14 +685,16 @@ def test_intensity_observes_and_polarimetric_cals(
|
|
|
541
685
|
assert "MODSTATE_1" in task.scratch.tags(file)
|
|
542
686
|
|
|
543
687
|
|
|
544
|
-
def test_dark_readout_exp_time_picky_bud(
|
|
688
|
+
def test_dark_readout_exp_time_picky_bud(
|
|
689
|
+
parse_task_with_no_data, mocker, fake_gql_client, lamp_readout_exp_time
|
|
690
|
+
):
|
|
545
691
|
"""
|
|
546
692
|
:Given: Dataset where non-dark readout exp time values are missing from the set of dark IP frames.
|
|
547
693
|
:When: Parsing
|
|
548
694
|
:Then: The `DarkReadoutExpTimePickyBud` raises an error
|
|
549
695
|
"""
|
|
550
696
|
mocker.patch(
|
|
551
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
697
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
552
698
|
)
|
|
553
699
|
bad_readout_exp_time = lamp_readout_exp_time + 0.02
|
|
554
700
|
dummy_exp_time = 99.0
|