dkist-processing-vbi 1.22.0__py3-none-any.whl → 1.25.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dkist_processing_vbi/__init__.py +1 -0
- dkist_processing_vbi/config.py +1 -0
- dkist_processing_vbi/models/constants.py +1 -0
- dkist_processing_vbi/models/filter.py +1 -0
- dkist_processing_vbi/models/parameters.py +1 -0
- dkist_processing_vbi/models/tags.py +1 -0
- dkist_processing_vbi/parsers/mosaic_repeats.py +1 -0
- dkist_processing_vbi/parsers/spatial_step_pattern.py +1 -0
- dkist_processing_vbi/parsers/vbi_l0_fits_access.py +1 -0
- dkist_processing_vbi/parsers/vbi_l1_fits_access.py +1 -0
- dkist_processing_vbi/tasks/__init__.py +1 -0
- dkist_processing_vbi/tasks/assemble_movie.py +1 -0
- dkist_processing_vbi/tasks/dark.py +5 -6
- dkist_processing_vbi/tasks/gain.py +5 -4
- dkist_processing_vbi/tasks/make_movie_frames.py +5 -6
- dkist_processing_vbi/tasks/parse.py +1 -0
- dkist_processing_vbi/tasks/process_summit_processed.py +2 -1
- dkist_processing_vbi/tasks/quality_metrics.py +3 -2
- dkist_processing_vbi/tasks/science.py +5 -4
- dkist_processing_vbi/tasks/vbi_base.py +1 -0
- dkist_processing_vbi/tasks/write_l1.py +2 -1
- dkist_processing_vbi/tests/conftest.py +12 -0
- dkist_processing_vbi/tests/local_trial_workflows/l0_to_l1.py +0 -20
- dkist_processing_vbi/tests/test_assemble_movie.py +6 -4
- dkist_processing_vbi/tests/test_dark.py +3 -4
- dkist_processing_vbi/tests/test_gain.py +4 -5
- dkist_processing_vbi/tests/test_make_movie_frames.py +4 -5
- dkist_processing_vbi/tests/test_parse_l0.py +19 -14
- dkist_processing_vbi/tests/test_parse_summit.py +8 -6
- dkist_processing_vbi/tests/test_process_summit.py +6 -7
- dkist_processing_vbi/tests/test_science.py +4 -5
- dkist_processing_vbi/tests/test_vbi_constants.py +1 -2
- dkist_processing_vbi/tests/test_workflows.py +1 -0
- dkist_processing_vbi/tests/test_write_l1.py +4 -3
- dkist_processing_vbi/workflows/__init__.py +1 -0
- dkist_processing_vbi/workflows/l0_processing.py +1 -0
- dkist_processing_vbi/workflows/summit_data_processing.py +1 -0
- dkist_processing_vbi/workflows/trial_workflows.py +1 -0
- {dkist_processing_vbi-1.22.0.dist-info → dkist_processing_vbi-1.25.2.dist-info}/METADATA +214 -126
- dkist_processing_vbi-1.25.2.dist-info/RECORD +64 -0
- docs/conf.py +1 -0
- dkist_processing_vbi-1.22.0.dist-info/RECORD +0 -64
- {dkist_processing_vbi-1.22.0.dist-info → dkist_processing_vbi-1.25.2.dist-info}/WHEEL +0 -0
- {dkist_processing_vbi-1.22.0.dist-info → dkist_processing_vbi-1.25.2.dist-info}/top_level.txt +0 -0
dkist_processing_vbi/__init__.py
CHANGED
dkist_processing_vbi/config.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""VBI dark calibration task."""
|
|
2
|
+
|
|
2
3
|
from dkist_processing_common.codecs.fits import fits_array_decoder
|
|
3
4
|
from dkist_processing_common.codecs.fits import fits_array_encoder
|
|
4
5
|
from dkist_processing_common.models.task_name import TaskName
|
|
@@ -46,7 +47,7 @@ class DarkCalibration(VbiTaskBase, QualityMixin):
|
|
|
46
47
|
set(self.constants.gain_exposure_times + self.constants.observe_exposure_times)
|
|
47
48
|
)
|
|
48
49
|
logger.info(f"{target_exp_times = }")
|
|
49
|
-
with self.
|
|
50
|
+
with self.telemetry_span(
|
|
50
51
|
f"Calculating dark frames for {self.constants.num_spatial_steps} steps and {len(target_exp_times)} exp times",
|
|
51
52
|
):
|
|
52
53
|
total_dark_frames_used = 0
|
|
@@ -67,16 +68,14 @@ class DarkCalibration(VbiTaskBase, QualityMixin):
|
|
|
67
68
|
auto_squeeze=False,
|
|
68
69
|
)
|
|
69
70
|
|
|
70
|
-
with self.
|
|
71
|
-
f"Processing dark for {step = } and {exp_time = }"
|
|
72
|
-
):
|
|
71
|
+
with self.telemetry_span(f"Processing dark for {step = } and {exp_time = }"):
|
|
73
72
|
logger.info(f"averaging arrays for step {step}")
|
|
74
73
|
averaged_dark_array = average_numpy_arrays(input_dark_arrays)
|
|
75
74
|
logger.info(
|
|
76
75
|
f"average dark signal in step {step} = {averaged_dark_array.mean():.3e}"
|
|
77
76
|
)
|
|
78
77
|
|
|
79
|
-
with self.
|
|
78
|
+
with self.telemetry_span(
|
|
80
79
|
f"Writing intermediate dark for {step = } and {exp_time = }",
|
|
81
80
|
):
|
|
82
81
|
self.write(
|
|
@@ -88,7 +87,7 @@ class DarkCalibration(VbiTaskBase, QualityMixin):
|
|
|
88
87
|
encoder=fits_array_encoder,
|
|
89
88
|
)
|
|
90
89
|
|
|
91
|
-
with self.
|
|
90
|
+
with self.telemetry_span("Computing and logging quality metrics"):
|
|
92
91
|
no_of_raw_dark_frames: int = self.count(tags=[VbiTag.input(), VbiTag.task_dark_frame()])
|
|
93
92
|
unused_count = no_of_raw_dark_frames - total_dark_frames_used
|
|
94
93
|
self.quality_store_task_type_counts(
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""VBI gain task."""
|
|
2
|
+
|
|
2
3
|
import numpy as np
|
|
3
4
|
from dkist_processing_common.codecs.fits import fits_array_decoder
|
|
4
5
|
from dkist_processing_common.codecs.fits import fits_array_encoder
|
|
@@ -52,7 +53,7 @@ class GainCalibration(VbiTaskBase, QualityMixin):
|
|
|
52
53
|
# This is OK (tm) because this will be, at most, 9 4k x 4k arrays. This is a lot (~1G), but not too much.
|
|
53
54
|
step_gain_dict: dict = {}
|
|
54
55
|
|
|
55
|
-
with self.
|
|
56
|
+
with self.telemetry_span(
|
|
56
57
|
f"Collecting and reducing gain arrays from {self.constants.num_spatial_steps} steps and {len(self.constants.gain_exposure_times)} exp times",
|
|
57
58
|
):
|
|
58
59
|
for exp_time in self.constants.gain_exposure_times:
|
|
@@ -99,13 +100,13 @@ class GainCalibration(VbiTaskBase, QualityMixin):
|
|
|
99
100
|
self.total_counts += np.nansum(dark_subtracted_gain_array)
|
|
100
101
|
step_gain_dict[step] = dark_subtracted_gain_array
|
|
101
102
|
|
|
102
|
-
with self.
|
|
103
|
+
with self.telemetry_span("normalizing gain arrays"):
|
|
103
104
|
normalized_array_dict = self.normalize_fov(step_gain_dict)
|
|
104
105
|
|
|
105
|
-
with self.
|
|
106
|
+
with self.telemetry_span("writing gain arrays to disk"):
|
|
106
107
|
self.write_gain_calibration(normalized_array_dict)
|
|
107
108
|
|
|
108
|
-
with self.
|
|
109
|
+
with self.telemetry_span("Computing and logging quality metrics"):
|
|
109
110
|
no_of_raw_gain_frames: int = self.count(
|
|
110
111
|
tags=[
|
|
111
112
|
VbiTag.input(),
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""VBI movie frame creation."""
|
|
2
|
+
|
|
2
3
|
import numpy as np
|
|
3
4
|
import scipy.ndimage as spnd
|
|
4
5
|
from astropy.io import fits
|
|
@@ -43,15 +44,15 @@ class MakeVbiMovieFrames(VbiTaskBase):
|
|
|
43
44
|
None
|
|
44
45
|
|
|
45
46
|
"""
|
|
46
|
-
with self.
|
|
47
|
+
with self.telemetry_span("averaging exposures"):
|
|
47
48
|
self.average_all_exposures()
|
|
48
49
|
|
|
49
|
-
with self.
|
|
50
|
+
with self.telemetry_span("stitching full FOV frames"):
|
|
50
51
|
for mosaic in range(1, self.constants.num_mosaic_repeats + 1):
|
|
51
52
|
logger.info(f"stitching full FOV for mosaic repeat {mosaic}")
|
|
52
53
|
output_hdl = self.stitch_single_mosaic_repeat(mosaic)
|
|
53
54
|
|
|
54
|
-
with self.
|
|
55
|
+
with self.telemetry_span(
|
|
55
56
|
f"writing stitched movie frame for mosaic repeat {mosaic}"
|
|
56
57
|
):
|
|
57
58
|
self.write(
|
|
@@ -144,9 +145,7 @@ class MakeVbiMovieFrames(VbiTaskBase):
|
|
|
144
145
|
output = np.zeros((size_y, size_x))
|
|
145
146
|
px_count = np.zeros((size_y, size_x))
|
|
146
147
|
|
|
147
|
-
with self.
|
|
148
|
-
f"stitching all camera positions together for {mosaic_repeat=}"
|
|
149
|
-
):
|
|
148
|
+
with self.telemetry_span(f"stitching all camera positions together for {mosaic_repeat=}"):
|
|
150
149
|
for o in all_step_access:
|
|
151
150
|
logger.info(f"Placing position {o.current_spatial_step} into full frame")
|
|
152
151
|
self.place_pos_in_full_fov(o, ref_header, output, px_count)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Repackage VBI data already calibrated before receipt at the Data Center."""
|
|
2
|
+
|
|
2
3
|
from astropy.io import fits
|
|
3
4
|
from dkist_processing_common.codecs.fits import fits_hdulist_encoder
|
|
4
5
|
|
|
@@ -33,7 +34,7 @@ class GenerateL1SummitData(VbiTaskBase):
|
|
|
33
34
|
- Write out
|
|
34
35
|
"""
|
|
35
36
|
# This loop is how we ensure that only completed mosaics get processed.
|
|
36
|
-
with self.
|
|
37
|
+
with self.telemetry_span("Re-tagging INPUT observe frames as CALIBRATED"):
|
|
37
38
|
for mosaic in range(1, self.constants.num_mosaic_repeats + 1):
|
|
38
39
|
for step in range(1, self.constants.num_spatial_steps + 1):
|
|
39
40
|
for file_name in self.read(
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""VBI specific quality metrics."""
|
|
2
|
+
|
|
2
3
|
from typing import Iterable
|
|
3
4
|
|
|
4
5
|
from dkist_processing_common.codecs.fits import fits_access_decoder
|
|
@@ -61,10 +62,10 @@ class VbiQualityL1Metrics(VbiTaskBase, QualityMixin):
|
|
|
61
62
|
)
|
|
62
63
|
datetimes = []
|
|
63
64
|
noise_values = []
|
|
64
|
-
with self.
|
|
65
|
+
with self.telemetry_span("Calculating VBI L1 quality metrics"):
|
|
65
66
|
for frame in frames:
|
|
66
67
|
datetimes.append(frame.time_obs)
|
|
67
68
|
noise_values.append(self.avg_noise(frame.data))
|
|
68
69
|
|
|
69
|
-
with self.
|
|
70
|
+
with self.telemetry_span("Sending lists for storage"):
|
|
70
71
|
self.quality_store_noise(datetimes=datetimes, values=noise_values)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""VBI science task."""
|
|
2
|
+
|
|
2
3
|
from astropy.io import fits
|
|
3
4
|
from dkist_processing_common.codecs.fits import fits_access_decoder
|
|
4
5
|
from dkist_processing_common.codecs.fits import fits_array_decoder
|
|
@@ -48,7 +49,7 @@ class ScienceCalibration(VbiTaskBase, QualityMixin):
|
|
|
48
49
|
logger.info(
|
|
49
50
|
f"Starting science with {self.constants.num_spatial_steps} steps and {self.constants.num_mosaic_repeats} mosaic repeats"
|
|
50
51
|
)
|
|
51
|
-
with self.
|
|
52
|
+
with self.telemetry_span(
|
|
52
53
|
f"Reducing science frames from {self.constants.num_spatial_steps} steps and {self.constants.num_mosaic_repeats} mosaic repeats",
|
|
53
54
|
):
|
|
54
55
|
for exp_time in self.constants.observe_exposure_times:
|
|
@@ -88,7 +89,7 @@ class ScienceCalibration(VbiTaskBase, QualityMixin):
|
|
|
88
89
|
auto_squeeze=False,
|
|
89
90
|
)
|
|
90
91
|
|
|
91
|
-
with self.
|
|
92
|
+
with self.telemetry_span("dark and gain corrections"):
|
|
92
93
|
logger.info(f"subtracting dark from {apm_str}")
|
|
93
94
|
sci_access = subtract_array_from_fits_access(
|
|
94
95
|
access_objs=sci_access, array_to_subtract=dark_calibration_array
|
|
@@ -99,13 +100,13 @@ class ScienceCalibration(VbiTaskBase, QualityMixin):
|
|
|
99
100
|
access_objs=sci_access, array_to_divide_by=gain_calibration_array
|
|
100
101
|
)
|
|
101
102
|
|
|
102
|
-
with self.
|
|
103
|
+
with self.telemetry_span("writing calibrated science frames"):
|
|
103
104
|
for i, access_obj in enumerate(sci_access):
|
|
104
105
|
exp_num = access_obj.current_mosaic_step_exp
|
|
105
106
|
logger.info(f"Writing output for {apm_str} and {exp_num = }")
|
|
106
107
|
self.write_calibrated_fits_obj(access_obj, mosaic, step)
|
|
107
108
|
|
|
108
|
-
with self.
|
|
109
|
+
with self.telemetry_span("Computing and logging quality metrics"):
|
|
109
110
|
no_of_raw_obs_frames: int = self.count(
|
|
110
111
|
tags=[
|
|
111
112
|
VbiTag.input(),
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""VBI Write L1 task."""
|
|
2
|
+
|
|
2
3
|
from typing import Literal
|
|
3
4
|
from typing import Type
|
|
4
5
|
|
|
@@ -13,8 +14,8 @@ from dkist_processing_common.tasks.write_l1 import WriteL1Frame
|
|
|
13
14
|
from dkist_service_configuration.logging import logger
|
|
14
15
|
|
|
15
16
|
from dkist_processing_vbi.models.constants import VbiConstants
|
|
16
|
-
from dkist_processing_vbi.models.filter import find_associated_filter
|
|
17
17
|
from dkist_processing_vbi.models.filter import VBI_FILTERS
|
|
18
|
+
from dkist_processing_vbi.models.filter import find_associated_filter
|
|
18
19
|
|
|
19
20
|
__all__ = ["VbiWriteL1Frame"]
|
|
20
21
|
|
|
@@ -16,6 +16,7 @@ from dkist_header_validator.translator import translate_spec122_to_spec214_l0
|
|
|
16
16
|
from dkist_processing_common.codecs.basemodel import basemodel_encoder
|
|
17
17
|
from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
|
|
18
18
|
from dkist_processing_common.models.task_name import TaskName
|
|
19
|
+
from dkist_processing_common.tests.mock_metadata_store import fake_gql_client
|
|
19
20
|
|
|
20
21
|
from dkist_processing_vbi.models.constants import VbiConstants
|
|
21
22
|
from dkist_processing_vbi.models.parameters import VbiParameters
|
|
@@ -95,6 +96,10 @@ class VbiS122Headers(Spec122Dataset):
|
|
|
95
96
|
self.add_constant_key("VBI__003", num_steps)
|
|
96
97
|
self.add_constant_key("VBI__007", num_exp_per_step)
|
|
97
98
|
self.add_constant_key("DKIST008", DKIST008_value)
|
|
99
|
+
self.add_constant_key("ID___014", "v1") # hls_version
|
|
100
|
+
self.add_constant_key("TELTRACK", "Fixed Solar Rotation Tracking")
|
|
101
|
+
self.add_constant_key("TTBLTRCK", "fixed angle on sun")
|
|
102
|
+
self.add_constant_key("TELSCAN", "Raster")
|
|
98
103
|
|
|
99
104
|
@key_function("VBI__004")
|
|
100
105
|
def spatial_step(self, key: str) -> int:
|
|
@@ -122,6 +127,13 @@ class Vbi122DarkFrames(VbiS122Headers):
|
|
|
122
127
|
array_shape, num_steps=num_steps, num_exp_per_step=num_exp_per_step, num_dsps_repeats=1
|
|
123
128
|
)
|
|
124
129
|
self.add_constant_key("DKIST004", TaskName.dark.value)
|
|
130
|
+
self.add_constant_key("PAC__002", "clear")
|
|
131
|
+
self.add_constant_key("PAC__003", "off")
|
|
132
|
+
self.add_constant_key("PAC__004", "clear")
|
|
133
|
+
self.add_constant_key("PAC__005", "10.")
|
|
134
|
+
self.add_constant_key("PAC__006", "clear")
|
|
135
|
+
self.add_constant_key("PAC__007", "20.")
|
|
136
|
+
self.add_constant_key("PAC__008", "FieldStop (5arcmin)")
|
|
125
137
|
|
|
126
138
|
|
|
127
139
|
class Vbi122GainFrames(VbiS122Headers):
|
|
@@ -146,21 +146,6 @@ class ValidateL1Output(VbiTaskBase):
|
|
|
146
146
|
spec214_validator.validate(f, extra=False)
|
|
147
147
|
|
|
148
148
|
|
|
149
|
-
def setup_APM_config() -> None:
|
|
150
|
-
mesh_config = {
|
|
151
|
-
"system-monitoring-log-apm": {
|
|
152
|
-
"mesh_address": "system-monitoring-log-apm.service.sim.consul",
|
|
153
|
-
"mesh_port": 8200,
|
|
154
|
-
},
|
|
155
|
-
"automated-processing-scratch-inventory": {"mesh_address": "localhost", "mesh_port": 6379},
|
|
156
|
-
"internal-api-gateway": {"mesh_address": "localhost", "mesh_port": 80},
|
|
157
|
-
}
|
|
158
|
-
apm_options = {"TRANSACTION_MAX_SPANS": 10000}
|
|
159
|
-
os.environ["MESH_CONFIG"] = json.dumps(mesh_config)
|
|
160
|
-
os.environ["ELASTIC_APM_ENABLED"] = "true"
|
|
161
|
-
os.environ["ELASTIC_APM_OTHER_OPTIONS"] = json.dumps(apm_options)
|
|
162
|
-
|
|
163
|
-
|
|
164
149
|
def l0_pipeline_workflow(manual_processing_run: ManualProcessing) -> None:
|
|
165
150
|
manual_processing_run.run_task(task=ShowExposureTimes)
|
|
166
151
|
manual_processing_run.run_task(task=VbiQualityL0Metrics)
|
|
@@ -181,10 +166,7 @@ def main(
|
|
|
181
166
|
skip_movie: bool = False,
|
|
182
167
|
only_translate: bool = False,
|
|
183
168
|
science_workflow_name: str = "l0_processing",
|
|
184
|
-
use_apm: bool = False,
|
|
185
169
|
):
|
|
186
|
-
if use_apm:
|
|
187
|
-
setup_APM_config()
|
|
188
170
|
science_func_dict = {
|
|
189
171
|
"l0_pipeline": l0_pipeline_workflow,
|
|
190
172
|
"summit_data_processed": summit_data_processing_workflow,
|
|
@@ -265,7 +247,6 @@ if __name__ == "__main__":
|
|
|
265
247
|
"-t", "--only-translate", help="Do ONLY the translation step", action="store_true"
|
|
266
248
|
)
|
|
267
249
|
parser.add_argument("-M", "--skip-movie", help="Skip making output movie", action="store_true")
|
|
268
|
-
parser.add_argument("-A", "--use-apm", help="Send APM spans to SIM", action="store_true")
|
|
269
250
|
args = parser.parse_args()
|
|
270
251
|
|
|
271
252
|
sys.exit(
|
|
@@ -277,6 +258,5 @@ if __name__ == "__main__":
|
|
|
277
258
|
only_translate=args.only_translate,
|
|
278
259
|
skip_movie=args.skip_movie,
|
|
279
260
|
science_workflow_name=args.workflow_name,
|
|
280
|
-
use_apm=args.use_apm,
|
|
281
261
|
)
|
|
282
262
|
)
|
|
@@ -2,14 +2,13 @@ import numpy as np
|
|
|
2
2
|
import pytest
|
|
3
3
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
4
4
|
from dkist_processing_common.codecs.fits import fits_hdulist_encoder
|
|
5
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
6
5
|
|
|
7
6
|
from dkist_processing_vbi.models.tags import VbiTag
|
|
8
7
|
from dkist_processing_vbi.tasks.assemble_movie import AssembleVbiMovie
|
|
9
|
-
from dkist_processing_vbi.tests.conftest import generate_214_l1_fits_frame
|
|
10
8
|
from dkist_processing_vbi.tests.conftest import Vbi122ObserveFrames
|
|
11
9
|
from dkist_processing_vbi.tests.conftest import VbiConstantsDb
|
|
12
10
|
from dkist_processing_vbi.tests.conftest import VbiInputDatasetParameterValues
|
|
11
|
+
from dkist_processing_vbi.tests.conftest import generate_214_l1_fits_frame
|
|
13
12
|
|
|
14
13
|
|
|
15
14
|
@pytest.fixture(scope="function")
|
|
@@ -49,10 +48,13 @@ def assemble_task_with_tagged_movie_frames(tmp_path, recipe_run_id, init_vbi_con
|
|
|
49
48
|
|
|
50
49
|
|
|
51
50
|
def test_assemble_movie(
|
|
52
|
-
assemble_task_with_tagged_movie_frames,
|
|
51
|
+
assemble_task_with_tagged_movie_frames,
|
|
52
|
+
mocker,
|
|
53
|
+
assign_input_dataset_doc_to_task,
|
|
54
|
+
fake_gql_client,
|
|
53
55
|
):
|
|
54
56
|
mocker.patch(
|
|
55
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
57
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
56
58
|
)
|
|
57
59
|
task = assemble_task_with_tagged_movie_frames
|
|
58
60
|
assign_input_dataset_doc_to_task(task, VbiInputDatasetParameterValues())
|
|
@@ -7,13 +7,12 @@ from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
|
7
7
|
from dkist_processing_common.codecs.fits import fits_array_decoder
|
|
8
8
|
from dkist_processing_common.codecs.fits import fits_array_encoder
|
|
9
9
|
from dkist_processing_common.models.tags import Tag
|
|
10
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
11
10
|
|
|
12
11
|
from dkist_processing_vbi.models.tags import VbiTag
|
|
13
12
|
from dkist_processing_vbi.tasks.dark import DarkCalibration
|
|
14
|
-
from dkist_processing_vbi.tests.conftest import ensure_all_inputs_used
|
|
15
13
|
from dkist_processing_vbi.tests.conftest import Vbi122DarkFrames
|
|
16
14
|
from dkist_processing_vbi.tests.conftest import VbiConstantsDb
|
|
15
|
+
from dkist_processing_vbi.tests.conftest import ensure_all_inputs_used
|
|
17
16
|
|
|
18
17
|
|
|
19
18
|
@pytest.fixture(scope="function")
|
|
@@ -65,14 +64,14 @@ def dark_calibration_task(tmp_path, recipe_run_id, init_vbi_constants_db):
|
|
|
65
64
|
task._purge()
|
|
66
65
|
|
|
67
66
|
|
|
68
|
-
def test_dark_calibration_task(dark_calibration_task, mocker):
|
|
67
|
+
def test_dark_calibration_task(dark_calibration_task, mocker, fake_gql_client):
|
|
69
68
|
"""
|
|
70
69
|
Given: a set of parsed input dark frames and a DarkCalibration task
|
|
71
70
|
When: running the task
|
|
72
71
|
Then: a single output array is produced for each spatial step and the array values are correct
|
|
73
72
|
"""
|
|
74
73
|
mocker.patch(
|
|
75
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
74
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
76
75
|
)
|
|
77
76
|
dark_calibration_task()
|
|
78
77
|
|
|
@@ -7,14 +7,13 @@ from dkist_processing_common.codecs.fits import fits_array_decoder
|
|
|
7
7
|
from dkist_processing_common.codecs.fits import fits_array_encoder
|
|
8
8
|
from dkist_processing_common.codecs.fits import fits_hdulist_encoder
|
|
9
9
|
from dkist_processing_common.models.tags import Tag
|
|
10
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
11
10
|
|
|
12
11
|
from dkist_processing_vbi.models.tags import VbiTag
|
|
13
12
|
from dkist_processing_vbi.tasks.gain import GainCalibration
|
|
14
|
-
from dkist_processing_vbi.tests.conftest import ensure_all_inputs_used
|
|
15
|
-
from dkist_processing_vbi.tests.conftest import generate_214_l0_fits_frame
|
|
16
13
|
from dkist_processing_vbi.tests.conftest import Vbi122GainFrames
|
|
17
14
|
from dkist_processing_vbi.tests.conftest import VbiConstantsDb
|
|
15
|
+
from dkist_processing_vbi.tests.conftest import ensure_all_inputs_used
|
|
16
|
+
from dkist_processing_vbi.tests.conftest import generate_214_l0_fits_frame
|
|
18
17
|
|
|
19
18
|
|
|
20
19
|
@pytest.fixture(scope="function")
|
|
@@ -65,14 +64,14 @@ def gain_calibration_task(tmp_path, recipe_run_id, init_vbi_constants_db):
|
|
|
65
64
|
task._purge()
|
|
66
65
|
|
|
67
66
|
|
|
68
|
-
def test_gain_calibration(gain_calibration_task, mocker):
|
|
67
|
+
def test_gain_calibration(gain_calibration_task, mocker, fake_gql_client):
|
|
69
68
|
"""
|
|
70
69
|
Given: a set of parsed input gain frames, dark calibration frames, and a GainCalibration task
|
|
71
70
|
When: the task is run
|
|
72
71
|
Then: a single array is produced for each step and the array values are correctly normalized
|
|
73
72
|
"""
|
|
74
73
|
mocker.patch(
|
|
75
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
74
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
76
75
|
)
|
|
77
76
|
gain_calibration_task()
|
|
78
77
|
|
|
@@ -9,16 +9,15 @@ from dkist_processing_common.codecs.fits import fits_access_decoder
|
|
|
9
9
|
from dkist_processing_common.codecs.fits import fits_array_decoder
|
|
10
10
|
from dkist_processing_common.codecs.fits import fits_hdulist_encoder
|
|
11
11
|
from dkist_processing_common.models.fits_access import FitsAccessBase
|
|
12
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
13
12
|
|
|
14
13
|
from dkist_processing_vbi.models.constants import VbiConstants
|
|
15
14
|
from dkist_processing_vbi.models.tags import VbiTag
|
|
16
15
|
from dkist_processing_vbi.parsers.vbi_l1_fits_access import VbiL1FitsAccess
|
|
17
16
|
from dkist_processing_vbi.tasks.make_movie_frames import MakeVbiMovieFrames
|
|
18
|
-
from dkist_processing_vbi.tests.conftest import ensure_all_inputs_used
|
|
19
|
-
from dkist_processing_vbi.tests.conftest import generate_214_l1_fits_frame
|
|
20
17
|
from dkist_processing_vbi.tests.conftest import Vbi122ObserveFrames
|
|
21
18
|
from dkist_processing_vbi.tests.conftest import VbiConstantsDb
|
|
19
|
+
from dkist_processing_vbi.tests.conftest import ensure_all_inputs_used
|
|
20
|
+
from dkist_processing_vbi.tests.conftest import generate_214_l1_fits_frame
|
|
22
21
|
|
|
23
22
|
|
|
24
23
|
@pytest.fixture(scope="function")
|
|
@@ -178,9 +177,9 @@ def compute_expected_mosiac(modification: int):
|
|
|
178
177
|
return expected
|
|
179
178
|
|
|
180
179
|
|
|
181
|
-
def test_make_movie_frames_task(raw_make_movie_frames_task, mocker):
|
|
180
|
+
def test_make_movie_frames_task(raw_make_movie_frames_task, mocker, fake_gql_client):
|
|
182
181
|
mocker.patch(
|
|
183
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
182
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
184
183
|
)
|
|
185
184
|
|
|
186
185
|
raw_make_movie_frames_task()
|
|
@@ -4,15 +4,14 @@ import pytest
|
|
|
4
4
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
5
5
|
from dkist_processing_common.codecs.fits import fits_hdulist_encoder
|
|
6
6
|
from dkist_processing_common.models.constants import BudName
|
|
7
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
8
7
|
|
|
9
8
|
from dkist_processing_vbi.models.constants import VbiBudName
|
|
10
9
|
from dkist_processing_vbi.models.tags import VbiTag
|
|
11
10
|
from dkist_processing_vbi.tasks.parse import ParseL0VbiInputData
|
|
12
|
-
from dkist_processing_vbi.tests.conftest import generate_214_l0_fits_frame
|
|
13
11
|
from dkist_processing_vbi.tests.conftest import Vbi122DarkFrames
|
|
14
12
|
from dkist_processing_vbi.tests.conftest import Vbi122GainFrames
|
|
15
13
|
from dkist_processing_vbi.tests.conftest import Vbi122ObserveFrames
|
|
14
|
+
from dkist_processing_vbi.tests.conftest import generate_214_l0_fits_frame
|
|
16
15
|
|
|
17
16
|
|
|
18
17
|
@pytest.fixture(scope="function")
|
|
@@ -181,14 +180,14 @@ def parse_inputs_task_with_aborted_last_mosaic(tmp_path, recipe_run_id):
|
|
|
181
180
|
task._purge()
|
|
182
181
|
|
|
183
182
|
|
|
184
|
-
def test_parse_l0_input_data_spatial_pos(parse_inputs_task, mocker):
|
|
183
|
+
def test_parse_l0_input_data_spatial_pos(parse_inputs_task, mocker, fake_gql_client):
|
|
185
184
|
"""
|
|
186
185
|
Given: a set of raw inputs of multiple task types and a ParseL0VbiInputData task
|
|
187
186
|
When: the task is run
|
|
188
187
|
Then: the input frames are correctly tagged by spatial position
|
|
189
188
|
"""
|
|
190
189
|
mocker.patch(
|
|
191
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
190
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
192
191
|
)
|
|
193
192
|
parse_inputs_task()
|
|
194
193
|
|
|
@@ -208,14 +207,14 @@ def test_parse_l0_input_data_spatial_pos(parse_inputs_task, mocker):
|
|
|
208
207
|
assert filepath.exists()
|
|
209
208
|
|
|
210
209
|
|
|
211
|
-
def test_parse_l0_input_constants(parse_inputs_task, mocker):
|
|
210
|
+
def test_parse_l0_input_constants(parse_inputs_task, mocker, fake_gql_client):
|
|
212
211
|
"""
|
|
213
212
|
Given: a set of raw inputs of multiple task types and a ParseL0VbiInputData task
|
|
214
213
|
When: the task is run
|
|
215
214
|
Then: pipeline constants are correctly updated from the input headers
|
|
216
215
|
"""
|
|
217
216
|
mocker.patch(
|
|
218
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
217
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
219
218
|
)
|
|
220
219
|
parse_inputs_task()
|
|
221
220
|
|
|
@@ -231,14 +230,14 @@ def test_parse_l0_input_constants(parse_inputs_task, mocker):
|
|
|
231
230
|
assert BudName.obs_ip_start_time.value in parse_inputs_task.constants._db_dict
|
|
232
231
|
|
|
233
232
|
|
|
234
|
-
def test_parse_l0_input_frames_found(parse_inputs_task, mocker):
|
|
233
|
+
def test_parse_l0_input_frames_found(parse_inputs_task, mocker, fake_gql_client):
|
|
235
234
|
"""
|
|
236
235
|
Given: a set of raw inputs of multiple task types and a ParseL0VbiInputData task
|
|
237
236
|
When: the task is run
|
|
238
237
|
Then: the frames from each task type are correctly identified and tagged
|
|
239
238
|
"""
|
|
240
239
|
mocker.patch(
|
|
241
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
240
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
242
241
|
)
|
|
243
242
|
parse_inputs_task()
|
|
244
243
|
assert (
|
|
@@ -258,7 +257,9 @@ def test_parse_l0_input_frames_found(parse_inputs_task, mocker):
|
|
|
258
257
|
)
|
|
259
258
|
|
|
260
259
|
|
|
261
|
-
def test_parse_l0_input_with_only_observe(
|
|
260
|
+
def test_parse_l0_input_with_only_observe(
|
|
261
|
+
parse_inputs_task_with_only_observe, mocker, fake_gql_client
|
|
262
|
+
):
|
|
262
263
|
"""
|
|
263
264
|
Given: a set of raw inputs of a single task type and a ParseL0VbiInputData task
|
|
264
265
|
When: the task is run
|
|
@@ -266,7 +267,7 @@ def test_parse_l0_input_with_only_observe(parse_inputs_task_with_only_observe, m
|
|
|
266
267
|
"""
|
|
267
268
|
task, camera_str = parse_inputs_task_with_only_observe
|
|
268
269
|
mocker.patch(
|
|
269
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
270
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
270
271
|
)
|
|
271
272
|
task()
|
|
272
273
|
if camera_str == "red":
|
|
@@ -293,14 +294,16 @@ def test_parse_l0_input_with_only_observe(parse_inputs_task_with_only_observe, m
|
|
|
293
294
|
assert filepath.exists()
|
|
294
295
|
|
|
295
296
|
|
|
296
|
-
def test_parse_l0_aborted_last_mosaic(
|
|
297
|
+
def test_parse_l0_aborted_last_mosaic(
|
|
298
|
+
parse_inputs_task_with_aborted_last_mosaic, mocker, fake_gql_client
|
|
299
|
+
):
|
|
297
300
|
"""
|
|
298
301
|
Given: a set of raw inputs representing a dataset with an aborted last mosaic
|
|
299
302
|
When: the task is run
|
|
300
303
|
Then: pipeline constants are correctly updated from the input headers
|
|
301
304
|
"""
|
|
302
305
|
mocker.patch(
|
|
303
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
306
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
304
307
|
)
|
|
305
308
|
task, expected_num_mosaic_repeats = parse_inputs_task_with_aborted_last_mosaic
|
|
306
309
|
task()
|
|
@@ -310,14 +313,16 @@ def test_parse_l0_aborted_last_mosaic(parse_inputs_task_with_aborted_last_mosaic
|
|
|
310
313
|
)
|
|
311
314
|
|
|
312
315
|
|
|
313
|
-
def test_parse_l0_correctly_tagged_mosaic_subrepeats(
|
|
316
|
+
def test_parse_l0_correctly_tagged_mosaic_subrepeats(
|
|
317
|
+
parse_inputs_task_with_subrepeats, mocker, fake_gql_client
|
|
318
|
+
):
|
|
314
319
|
"""
|
|
315
320
|
Given: A set of observe frames taken with subrepeats (i.e., multiple mosaics per DSPS repeat)
|
|
316
321
|
When: the parse task is run
|
|
317
322
|
Then: pipeline constants are correctly updated and the correct
|
|
318
323
|
"""
|
|
319
324
|
mocker.patch(
|
|
320
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=
|
|
325
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
321
326
|
)
|
|
322
327
|
task = parse_inputs_task_with_subrepeats
|
|
323
328
|
task()
|