dkist-processing-visp 3.3.0__py3-none-any.whl → 5.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dkist_processing_visp/__init__.py +1 -0
- dkist_processing_visp/config.py +1 -0
- dkist_processing_visp/models/constants.py +52 -21
- dkist_processing_visp/models/fits_access.py +20 -0
- dkist_processing_visp/models/metric_code.py +10 -0
- dkist_processing_visp/models/parameters.py +129 -19
- dkist_processing_visp/models/tags.py +1 -0
- dkist_processing_visp/models/task_name.py +1 -0
- dkist_processing_visp/parsers/map_repeats.py +1 -0
- dkist_processing_visp/parsers/modulator_states.py +1 -0
- dkist_processing_visp/parsers/polarimeter_mode.py +3 -1
- dkist_processing_visp/parsers/raster_step.py +4 -1
- dkist_processing_visp/parsers/spectrograph_configuration.py +75 -0
- dkist_processing_visp/parsers/time.py +15 -7
- dkist_processing_visp/parsers/visp_l0_fits_access.py +19 -8
- dkist_processing_visp/parsers/visp_l1_fits_access.py +1 -0
- dkist_processing_visp/tasks/__init__.py +1 -0
- dkist_processing_visp/tasks/assemble_movie.py +1 -0
- dkist_processing_visp/tasks/background_light.py +2 -1
- dkist_processing_visp/tasks/dark.py +5 -4
- dkist_processing_visp/tasks/geometric.py +132 -20
- dkist_processing_visp/tasks/instrument_polarization.py +13 -12
- dkist_processing_visp/tasks/l1_output_data.py +203 -0
- dkist_processing_visp/tasks/lamp.py +53 -93
- dkist_processing_visp/tasks/make_movie_frames.py +8 -6
- dkist_processing_visp/tasks/mixin/beam_access.py +1 -0
- dkist_processing_visp/tasks/mixin/corrections.py +54 -4
- dkist_processing_visp/tasks/mixin/downsample.py +1 -0
- dkist_processing_visp/tasks/parse.py +34 -4
- dkist_processing_visp/tasks/quality_metrics.py +5 -4
- dkist_processing_visp/tasks/science.py +126 -46
- dkist_processing_visp/tasks/solar.py +896 -456
- dkist_processing_visp/tasks/visp_base.py +2 -0
- dkist_processing_visp/tasks/write_l1.py +25 -5
- dkist_processing_visp/tests/conftest.py +99 -35
- dkist_processing_visp/tests/header_models.py +92 -20
- dkist_processing_visp/tests/local_trial_workflows/l0_cals_only.py +4 -23
- dkist_processing_visp/tests/local_trial_workflows/l0_polcals_as_science.py +421 -0
- dkist_processing_visp/tests/local_trial_workflows/l0_solar_gain_as_science.py +10 -29
- dkist_processing_visp/tests/local_trial_workflows/l0_to_l1.py +1 -21
- dkist_processing_visp/tests/local_trial_workflows/local_trial_helpers.py +98 -14
- dkist_processing_visp/tests/test_assemble_movie.py +2 -3
- dkist_processing_visp/tests/test_assemble_quality.py +89 -4
- dkist_processing_visp/tests/test_background_light.py +8 -5
- dkist_processing_visp/tests/test_dark.py +4 -3
- dkist_processing_visp/tests/test_fits_access.py +43 -0
- dkist_processing_visp/tests/test_geometric.py +45 -4
- dkist_processing_visp/tests/test_instrument_polarization.py +4 -3
- dkist_processing_visp/tests/test_lamp.py +22 -26
- dkist_processing_visp/tests/test_make_movie_frames.py +4 -4
- dkist_processing_visp/tests/test_map_repeats.py +3 -1
- dkist_processing_visp/tests/test_parameters.py +122 -21
- dkist_processing_visp/tests/test_parse.py +98 -14
- dkist_processing_visp/tests/test_quality.py +2 -3
- dkist_processing_visp/tests/test_science.py +113 -15
- dkist_processing_visp/tests/test_solar.py +318 -99
- dkist_processing_visp/tests/test_visp_constants.py +36 -8
- dkist_processing_visp/tests/test_workflows.py +1 -0
- dkist_processing_visp/tests/test_write_l1.py +17 -3
- dkist_processing_visp/workflows/__init__.py +1 -0
- dkist_processing_visp/workflows/l0_processing.py +8 -2
- dkist_processing_visp/workflows/trial_workflows.py +8 -2
- dkist_processing_visp-5.1.1.dist-info/METADATA +552 -0
- dkist_processing_visp-5.1.1.dist-info/RECORD +94 -0
- docs/conf.py +5 -1
- docs/gain_correction.rst +50 -42
- dkist_processing_visp/tasks/mixin/line_zones.py +0 -115
- dkist_processing_visp-3.3.0.dist-info/METADATA +0 -459
- dkist_processing_visp-3.3.0.dist-info/RECORD +0 -90
- {dkist_processing_visp-3.3.0.dist-info → dkist_processing_visp-5.1.1.dist-info}/WHEEL +0 -0
- {dkist_processing_visp-3.3.0.dist-info → dkist_processing_visp-5.1.1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,421 @@
|
|
|
1
|
+
"""For running just PolCal frames as science data and saving their outputs."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
import sys
|
|
7
|
+
from dataclasses import asdict
|
|
8
|
+
from random import randint
|
|
9
|
+
from typing import Literal
|
|
10
|
+
|
|
11
|
+
from astropy.io import fits
|
|
12
|
+
from dkist_processing_common.codecs.basemodel import basemodel_encoder
|
|
13
|
+
from dkist_processing_common.manual import ManualProcessing
|
|
14
|
+
from dkist_processing_common.models.fits_access import MetadataKey
|
|
15
|
+
from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
|
|
16
|
+
from dkist_processing_common.tasks import CreateTrialQualityReport
|
|
17
|
+
from dkist_processing_common.tasks import QualityL1Metrics
|
|
18
|
+
from dkist_processing_common.tasks import WorkflowTaskBase
|
|
19
|
+
from dkist_service_configuration.logging import logger
|
|
20
|
+
|
|
21
|
+
from dkist_processing_visp.models.constants import VispBudName
|
|
22
|
+
from dkist_processing_visp.models.fits_access import VispMetadataKey
|
|
23
|
+
from dkist_processing_visp.models.tags import VispTag
|
|
24
|
+
from dkist_processing_visp.tasks import AssembleVispMovie
|
|
25
|
+
from dkist_processing_visp.tasks import MakeVispMovieFrames
|
|
26
|
+
from dkist_processing_visp.tasks.background_light import BackgroundLightCalibration
|
|
27
|
+
from dkist_processing_visp.tasks.dark import DarkCalibration
|
|
28
|
+
from dkist_processing_visp.tasks.geometric import GeometricCalibration
|
|
29
|
+
from dkist_processing_visp.tasks.instrument_polarization import InstrumentPolarizationCalibration
|
|
30
|
+
from dkist_processing_visp.tasks.l1_output_data import VispAssembleQualityData
|
|
31
|
+
from dkist_processing_visp.tasks.lamp import LampCalibration
|
|
32
|
+
from dkist_processing_visp.tasks.science import ScienceCalibration
|
|
33
|
+
from dkist_processing_visp.tasks.solar import SolarCalibration
|
|
34
|
+
from dkist_processing_visp.tasks.visp_base import VispTaskBase
|
|
35
|
+
from dkist_processing_visp.tasks.write_l1 import VispWriteL1Frame
|
|
36
|
+
from dkist_processing_visp.tests.conftest import VispInputDatasetParameterValues
|
|
37
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadBackgroundCal
|
|
38
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadCalibratedData
|
|
39
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadDarkCal
|
|
40
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadGeometricCal
|
|
41
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadInputParsing
|
|
42
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadInstPolCal
|
|
43
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadLampCal
|
|
44
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
45
|
+
LoadPolcalAsScience,
|
|
46
|
+
)
|
|
47
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadSolarCal
|
|
48
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
49
|
+
ParseCalOnlyL0InputData,
|
|
50
|
+
)
|
|
51
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveBackgroundCal
|
|
52
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveCalibratedData
|
|
53
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveDarkCal
|
|
54
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveGeometricCal
|
|
55
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveInputParsing
|
|
56
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveInstPolCal
|
|
57
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveLampCal
|
|
58
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
59
|
+
SavePolcalAsScience,
|
|
60
|
+
)
|
|
61
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveSolarCal
|
|
62
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SetAxesTypes
|
|
63
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
64
|
+
SetCadenceConstants,
|
|
65
|
+
)
|
|
66
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SetNumModstates
|
|
67
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SetObserveExpTime
|
|
68
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
69
|
+
SetObserveIpStartTime,
|
|
70
|
+
)
|
|
71
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SetPolarimeterMode
|
|
72
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import ValidateL1Output
|
|
73
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
74
|
+
set_observe_wavelength_task,
|
|
75
|
+
)
|
|
76
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import tag_inputs_task
|
|
77
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
78
|
+
translate_122_to_214l0_task,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
__version__ = "PCAS"
|
|
82
|
+
|
|
83
|
+
QUALITY = False
|
|
84
|
+
try:
|
|
85
|
+
import dkist_quality
|
|
86
|
+
|
|
87
|
+
QUALITY = True
|
|
88
|
+
except ModuleNotFoundError:
|
|
89
|
+
logging.warning("Could not find dkist-quality")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class CreateInputDatasetParameterDocument(WorkflowTaskBase):
|
|
93
|
+
def run(self) -> None:
|
|
94
|
+
relative_path = "input_dataset_parameters.json"
|
|
95
|
+
self.write(
|
|
96
|
+
data=InputDatasetPartDocumentList(
|
|
97
|
+
doc_list=self.input_dataset_document_simple_parameters_part
|
|
98
|
+
),
|
|
99
|
+
relative_path=relative_path,
|
|
100
|
+
tags=VispTag.input_dataset_parameters(),
|
|
101
|
+
encoder=basemodel_encoder,
|
|
102
|
+
overwrite=True,
|
|
103
|
+
)
|
|
104
|
+
logger.info(f"Wrote input dataset parameter doc to {relative_path}")
|
|
105
|
+
|
|
106
|
+
@property
|
|
107
|
+
def input_dataset_document_simple_parameters_part(self):
|
|
108
|
+
parameters_list = []
|
|
109
|
+
value_id = randint(1000, 2000)
|
|
110
|
+
for pn, pv in asdict(
|
|
111
|
+
VispInputDatasetParameterValues(
|
|
112
|
+
visp_background_on=False,
|
|
113
|
+
visp_geo_upsample_factor=10000,
|
|
114
|
+
visp_polcal_num_spatial_bins=2560,
|
|
115
|
+
)
|
|
116
|
+
).items():
|
|
117
|
+
values = [
|
|
118
|
+
{
|
|
119
|
+
"parameterValueId": value_id,
|
|
120
|
+
"parameterValue": json.dumps(pv),
|
|
121
|
+
"parameterValueStartDate": "1946-11-20",
|
|
122
|
+
}
|
|
123
|
+
]
|
|
124
|
+
parameter = {"parameterName": pn, "parameterValues": values}
|
|
125
|
+
parameters_list.append(parameter)
|
|
126
|
+
|
|
127
|
+
return parameters_list
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class TagPolcalAsScience(VispTaskBase):
|
|
131
|
+
"""Do."""
|
|
132
|
+
|
|
133
|
+
def run(self) -> None:
|
|
134
|
+
"""Do."""
|
|
135
|
+
# First, tag the polcal frames as observe frames with the correct stuff
|
|
136
|
+
num_raster_set = set()
|
|
137
|
+
for cs_step in range(self.constants.num_cs_steps):
|
|
138
|
+
for modstate in range(1, self.constants.num_modstates + 1):
|
|
139
|
+
file_list = list(
|
|
140
|
+
self.read(
|
|
141
|
+
tags=[
|
|
142
|
+
VispTag.task_polcal(),
|
|
143
|
+
VispTag.cs_step(cs_step),
|
|
144
|
+
VispTag.modstate(modstate),
|
|
145
|
+
]
|
|
146
|
+
)
|
|
147
|
+
)
|
|
148
|
+
num_raster_set.add(len(file_list))
|
|
149
|
+
for raster_step, file_name in enumerate(file_list):
|
|
150
|
+
og_tags = self.tags(file_name)
|
|
151
|
+
logging.info(
|
|
152
|
+
f"Raw frame {file_name} at {cs_step = } and {modstate = } has {og_tags = }"
|
|
153
|
+
)
|
|
154
|
+
self.scratch._tag_db.clear_value(file_name)
|
|
155
|
+
logging.info(f"\tafter removing, the tags are {self.tags(file_name)}")
|
|
156
|
+
|
|
157
|
+
hdul = fits.open(file_name, mode="update")
|
|
158
|
+
idx = 0
|
|
159
|
+
if hdul[idx].data is None:
|
|
160
|
+
idx = 1
|
|
161
|
+
hdul[idx].header[VispMetadataKey.raster_scan_step] = raster_step
|
|
162
|
+
hdul.flush()
|
|
163
|
+
del hdul
|
|
164
|
+
|
|
165
|
+
new_tags = [
|
|
166
|
+
VispTag.task_observe(),
|
|
167
|
+
VispTag.map_scan(cs_step + 1),
|
|
168
|
+
VispTag.raster_step(raster_step),
|
|
169
|
+
VispTag.frame(),
|
|
170
|
+
VispTag.modstate(modstate),
|
|
171
|
+
VispTag.input(),
|
|
172
|
+
VispTag.readout_exp_time(self.constants.observe_readout_exp_times[0]),
|
|
173
|
+
]
|
|
174
|
+
logging.info(f"\tadding {new_tags = }")
|
|
175
|
+
self.tag(file_name, new_tags)
|
|
176
|
+
final_tags = self.tags(file_name)
|
|
177
|
+
logging.info(f"\tafter retagging tags are {final_tags}")
|
|
178
|
+
|
|
179
|
+
if len(num_raster_set) != 1:
|
|
180
|
+
raise ValueError(
|
|
181
|
+
"Expected to find the same number of files for each (CS step, modstate), but did not. "
|
|
182
|
+
f"Set of counts is {num_raster_set}"
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
num_raster_steps = num_raster_set.pop()
|
|
186
|
+
logging.info(f"Found {num_raster_steps} raster steps")
|
|
187
|
+
# Now update the num[map, raster] constants. We'll call each CS step a "map"
|
|
188
|
+
self.constants._update(
|
|
189
|
+
{
|
|
190
|
+
VispBudName.num_raster_steps.value: num_raster_steps,
|
|
191
|
+
VispBudName.num_map_scans.value: self.constants.num_cs_steps,
|
|
192
|
+
}
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def write_L1_files_task(prefix: str = ""):
|
|
197
|
+
"""Do."""
|
|
198
|
+
if len(prefix) > 0 and prefix[-1] != "_":
|
|
199
|
+
prefix += "_"
|
|
200
|
+
|
|
201
|
+
class WritePolcalL1Files(VispWriteL1Frame):
|
|
202
|
+
"""Do."""
|
|
203
|
+
|
|
204
|
+
def l1_filename(self, header: fits.Header, stokes: Literal["I", "Q", "U", "V"]):
|
|
205
|
+
"""Do."""
|
|
206
|
+
wavelength = str(round(header[MetadataKey.wavelength] * 1000)).zfill(8)
|
|
207
|
+
cs_step = header["VSPMAP"]
|
|
208
|
+
raster_step = header[VispMetadataKey.raster_scan_step]
|
|
209
|
+
return f"{prefix}CS_STEP_{cs_step:02n}_{raster_step:02n}_{wavelength}_{stokes}_L1.fits"
|
|
210
|
+
|
|
211
|
+
return WritePolcalL1Files
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def main(
|
|
215
|
+
scratch_path: str,
|
|
216
|
+
suffix: str = "fits",
|
|
217
|
+
prefix: str = "",
|
|
218
|
+
recipe_run_id: int = 2,
|
|
219
|
+
skip_translation: bool = False,
|
|
220
|
+
only_translate: bool = False,
|
|
221
|
+
load_parse: bool = False,
|
|
222
|
+
load_dark: bool = False,
|
|
223
|
+
load_background: bool = False,
|
|
224
|
+
load_lamp: bool = False,
|
|
225
|
+
load_geometric: bool = False,
|
|
226
|
+
load_solar: bool = False,
|
|
227
|
+
load_inst_pol: bool = False,
|
|
228
|
+
load_polcal_as_science: bool = False,
|
|
229
|
+
load_calibrated_data: bool = False,
|
|
230
|
+
dummy_wavelength: float = 630.0,
|
|
231
|
+
):
|
|
232
|
+
"""Run the damn thing."""
|
|
233
|
+
with ManualProcessing(
|
|
234
|
+
workflow_path=scratch_path,
|
|
235
|
+
recipe_run_id=recipe_run_id,
|
|
236
|
+
testing=True,
|
|
237
|
+
workflow_name="visp-l0-pipeline",
|
|
238
|
+
workflow_version="GROGU",
|
|
239
|
+
) as manual_processing_run:
|
|
240
|
+
if not skip_translation:
|
|
241
|
+
manual_processing_run.run_task(task=translate_122_to_214l0_task(suffix))
|
|
242
|
+
if only_translate:
|
|
243
|
+
return
|
|
244
|
+
manual_processing_run.run_task(task=CreateInputDatasetParameterDocument)
|
|
245
|
+
if load_parse:
|
|
246
|
+
manual_processing_run.run_task(task=LoadInputParsing)
|
|
247
|
+
else:
|
|
248
|
+
manual_processing_run.run_task(task=tag_inputs_task(suffix))
|
|
249
|
+
manual_processing_run.run_task(task=ParseCalOnlyL0InputData)
|
|
250
|
+
manual_processing_run.run_task(
|
|
251
|
+
task=set_observe_wavelength_task(wavelength=dummy_wavelength)
|
|
252
|
+
)
|
|
253
|
+
manual_processing_run.run_task(task=SetObserveIpStartTime)
|
|
254
|
+
manual_processing_run.run_task(task=SetObserveExpTime)
|
|
255
|
+
manual_processing_run.run_task(task=SetPolarimeterMode)
|
|
256
|
+
manual_processing_run.run_task(task=SetNumModstates)
|
|
257
|
+
manual_processing_run.run_task(task=SetCadenceConstants)
|
|
258
|
+
manual_processing_run.run_task(task=SetAxesTypes)
|
|
259
|
+
manual_processing_run.run_task(task=SaveInputParsing)
|
|
260
|
+
|
|
261
|
+
if load_dark:
|
|
262
|
+
manual_processing_run.run_task(task=LoadDarkCal)
|
|
263
|
+
else:
|
|
264
|
+
manual_processing_run.run_task(task=DarkCalibration)
|
|
265
|
+
manual_processing_run.run_task(task=SaveDarkCal)
|
|
266
|
+
|
|
267
|
+
if load_background:
|
|
268
|
+
manual_processing_run.run_task(task=LoadBackgroundCal)
|
|
269
|
+
else:
|
|
270
|
+
manual_processing_run.run_task(task=BackgroundLightCalibration)
|
|
271
|
+
manual_processing_run.run_task(task=SaveBackgroundCal)
|
|
272
|
+
|
|
273
|
+
if load_lamp:
|
|
274
|
+
manual_processing_run.run_task(task=LoadLampCal)
|
|
275
|
+
else:
|
|
276
|
+
manual_processing_run.run_task(task=LampCalibration)
|
|
277
|
+
manual_processing_run.run_task(task=SaveLampCal)
|
|
278
|
+
|
|
279
|
+
if load_geometric:
|
|
280
|
+
manual_processing_run.run_task(task=LoadGeometricCal)
|
|
281
|
+
else:
|
|
282
|
+
manual_processing_run.run_task(task=GeometricCalibration)
|
|
283
|
+
manual_processing_run.run_task(task=SaveGeometricCal)
|
|
284
|
+
|
|
285
|
+
if load_solar:
|
|
286
|
+
manual_processing_run.run_task(task=LoadSolarCal)
|
|
287
|
+
else:
|
|
288
|
+
manual_processing_run.run_task(task=SolarCalibration)
|
|
289
|
+
manual_processing_run.run_task(task=SaveSolarCal)
|
|
290
|
+
|
|
291
|
+
if load_inst_pol:
|
|
292
|
+
manual_processing_run.run_task(task=LoadInstPolCal)
|
|
293
|
+
else:
|
|
294
|
+
manual_processing_run.run_task(task=InstrumentPolarizationCalibration)
|
|
295
|
+
manual_processing_run.run_task(task=SaveInstPolCal)
|
|
296
|
+
|
|
297
|
+
if load_polcal_as_science:
|
|
298
|
+
manual_processing_run.run_task(task=LoadPolcalAsScience)
|
|
299
|
+
else:
|
|
300
|
+
manual_processing_run.run_task(task=TagPolcalAsScience)
|
|
301
|
+
manual_processing_run.run_task(task=SavePolcalAsScience)
|
|
302
|
+
|
|
303
|
+
if load_calibrated_data:
|
|
304
|
+
manual_processing_run.run_task(task=LoadCalibratedData)
|
|
305
|
+
else:
|
|
306
|
+
manual_processing_run.run_task(task=ScienceCalibration)
|
|
307
|
+
manual_processing_run.run_task(task=SaveCalibratedData)
|
|
308
|
+
|
|
309
|
+
manual_processing_run.run_task(task=write_L1_files_task(prefix=prefix))
|
|
310
|
+
manual_processing_run.run_task(task=QualityL1Metrics)
|
|
311
|
+
manual_processing_run.run_task(task=VispAssembleQualityData)
|
|
312
|
+
manual_processing_run.run_task(task=ValidateL1Output)
|
|
313
|
+
manual_processing_run.run_task(task=MakeVispMovieFrames)
|
|
314
|
+
manual_processing_run.run_task(task=AssembleVispMovie)
|
|
315
|
+
|
|
316
|
+
# Test some downstream services
|
|
317
|
+
if QUALITY:
|
|
318
|
+
manual_processing_run.run_task(task=CreateTrialQualityReport)
|
|
319
|
+
else:
|
|
320
|
+
logger.warning("Did NOT make quality report pdf because dkist-quality is not installed")
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
if __name__ == "__main__":
|
|
324
|
+
parser = argparse.ArgumentParser(
|
|
325
|
+
description="Generate PolCals with the ViSP DC Science pipeline"
|
|
326
|
+
)
|
|
327
|
+
parser.add_argument("scratch_path", help="Location to use as the DC 'scratch' disk")
|
|
328
|
+
parser.add_argument(
|
|
329
|
+
"-i",
|
|
330
|
+
"--run-id",
|
|
331
|
+
help="Which subdir to use. This will become the recipe run id",
|
|
332
|
+
type=int,
|
|
333
|
+
)
|
|
334
|
+
parser.add_argument("--prefix", help="File prefix to add to L1 output frames", default="")
|
|
335
|
+
parser.add_argument("--suffix", help="File suffix to treat as INPUT frames", default="fits")
|
|
336
|
+
parser.add_argument(
|
|
337
|
+
"-w",
|
|
338
|
+
"--wavelength",
|
|
339
|
+
help="Dummy wavelength to use for loading parameters, etc.",
|
|
340
|
+
type=float,
|
|
341
|
+
default=630.0,
|
|
342
|
+
)
|
|
343
|
+
parser.add_argument(
|
|
344
|
+
"-T",
|
|
345
|
+
"--skip-translation",
|
|
346
|
+
help="Skip the translation of raw 122 l0 frames to 214 l0",
|
|
347
|
+
action="store_true",
|
|
348
|
+
)
|
|
349
|
+
parser.add_argument(
|
|
350
|
+
"-t", "--only-translate", help="Do ONLY the translation step", action="store_true"
|
|
351
|
+
)
|
|
352
|
+
parser.add_argument(
|
|
353
|
+
"-I", "--load-input-parsing", help="Load tags on input files", action="store_true"
|
|
354
|
+
)
|
|
355
|
+
parser.add_argument(
|
|
356
|
+
"-D",
|
|
357
|
+
"--load-dark",
|
|
358
|
+
help="Load dark calibration from previously saved run",
|
|
359
|
+
action="store_true",
|
|
360
|
+
)
|
|
361
|
+
parser.add_argument(
|
|
362
|
+
"-B",
|
|
363
|
+
"--load-background",
|
|
364
|
+
help="Load background calibration from previously saved run",
|
|
365
|
+
action="store_true",
|
|
366
|
+
)
|
|
367
|
+
parser.add_argument(
|
|
368
|
+
"-L",
|
|
369
|
+
"--load-lamp",
|
|
370
|
+
help="Load lamp calibration from previously saved run",
|
|
371
|
+
action="store_true",
|
|
372
|
+
)
|
|
373
|
+
parser.add_argument(
|
|
374
|
+
"-G",
|
|
375
|
+
"--load-geometric",
|
|
376
|
+
help="Load geometric calibration from previously saved run",
|
|
377
|
+
action="store_true",
|
|
378
|
+
)
|
|
379
|
+
parser.add_argument(
|
|
380
|
+
"-S",
|
|
381
|
+
"--load-solar",
|
|
382
|
+
help="Load solar calibration from previously saved run",
|
|
383
|
+
action="store_true",
|
|
384
|
+
)
|
|
385
|
+
parser.add_argument(
|
|
386
|
+
"-P",
|
|
387
|
+
"--load-inst-pol",
|
|
388
|
+
help="Load instrument polarization calibration from previously saved run",
|
|
389
|
+
action="store_true",
|
|
390
|
+
)
|
|
391
|
+
parser.add_argument(
|
|
392
|
+
"-O",
|
|
393
|
+
"--load-polcal-as-science",
|
|
394
|
+
help="Don't re-make the polcal-as-science frames",
|
|
395
|
+
action="store_true",
|
|
396
|
+
)
|
|
397
|
+
parser.add_argument(
|
|
398
|
+
"-C", "--load-calibrated-data", help="Load CALIBRATED 'science' frames", action="store_true"
|
|
399
|
+
)
|
|
400
|
+
args = parser.parse_args()
|
|
401
|
+
logging.info(f"Called as {' '.join(sys.argv)}")
|
|
402
|
+
sys.exit(
|
|
403
|
+
main(
|
|
404
|
+
scratch_path=args.scratch_path,
|
|
405
|
+
suffix=args.suffix,
|
|
406
|
+
prefix=args.prefix,
|
|
407
|
+
recipe_run_id=args.run_id,
|
|
408
|
+
skip_translation=args.skip_translation,
|
|
409
|
+
only_translate=args.only_translate,
|
|
410
|
+
load_parse=args.load_input_parsing,
|
|
411
|
+
load_dark=args.load_dark,
|
|
412
|
+
load_background=args.load_background,
|
|
413
|
+
load_lamp=args.load_lamp,
|
|
414
|
+
load_geometric=args.load_geometric,
|
|
415
|
+
load_solar=args.load_solar,
|
|
416
|
+
load_inst_pol=args.load_inst_pol,
|
|
417
|
+
load_polcal_as_science=args.load_polcal_as_science,
|
|
418
|
+
load_calibrated_data=args.load_calibrated_data,
|
|
419
|
+
dummy_wavelength=args.wavelength,
|
|
420
|
+
)
|
|
421
|
+
)
|
|
@@ -29,9 +29,6 @@ from dkist_processing_visp.tasks.instrument_polarization import InstrumentPolari
|
|
|
29
29
|
from dkist_processing_visp.tasks.lamp import LampCalibration
|
|
30
30
|
from dkist_processing_visp.tasks.solar import SolarCalibration
|
|
31
31
|
from dkist_processing_visp.tests.conftest import VispInputDatasetParameterValues
|
|
32
|
-
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
33
|
-
load_solar_gain_as_science_task,
|
|
34
|
-
)
|
|
35
32
|
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadBackgroundCal
|
|
36
33
|
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadCalibratedData
|
|
37
34
|
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadDarkCal
|
|
@@ -54,9 +51,6 @@ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers impor
|
|
|
54
51
|
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
55
52
|
SaveSolarGainAsScience,
|
|
56
53
|
)
|
|
57
|
-
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
58
|
-
set_observe_wavelength_task,
|
|
59
|
-
)
|
|
60
54
|
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SetAxesTypes
|
|
61
55
|
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
62
56
|
SetCadenceConstants,
|
|
@@ -67,20 +61,26 @@ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers impor
|
|
|
67
61
|
SetObserveIpStartTime,
|
|
68
62
|
)
|
|
69
63
|
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SetPolarimeterMode
|
|
70
|
-
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import tag_inputs_task
|
|
71
64
|
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
72
65
|
TagModulatedSolarGainsAsScience,
|
|
73
66
|
)
|
|
74
67
|
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
75
68
|
TagSingleSolarGainAsScience,
|
|
76
69
|
)
|
|
70
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import ValidateL1Output
|
|
71
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
72
|
+
load_solar_gain_as_science_task,
|
|
73
|
+
)
|
|
74
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
75
|
+
set_observe_wavelength_task,
|
|
76
|
+
)
|
|
77
|
+
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import tag_inputs_task
|
|
77
78
|
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
78
79
|
transfer_trial_data_locally_task,
|
|
79
80
|
)
|
|
80
81
|
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
|
|
81
82
|
translate_122_to_214l0_task,
|
|
82
83
|
)
|
|
83
|
-
from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import ValidateL1Output
|
|
84
84
|
|
|
85
85
|
QUALITY = False
|
|
86
86
|
try:
|
|
@@ -118,6 +118,7 @@ class CreateInputDatasetParameterDocument(WorkflowTaskBase):
|
|
|
118
118
|
relative_path=relative_path,
|
|
119
119
|
tags=VispTag.input_dataset_parameters(),
|
|
120
120
|
encoder=basemodel_encoder,
|
|
121
|
+
overwrite=True,
|
|
121
122
|
)
|
|
122
123
|
logger.info(f"Wrote input dataset parameter doc to {relative_path}")
|
|
123
124
|
|
|
@@ -145,21 +146,6 @@ class CreateInputDatasetParameterDocument(WorkflowTaskBase):
|
|
|
145
146
|
return parameters_list
|
|
146
147
|
|
|
147
148
|
|
|
148
|
-
def setup_APM_config() -> None:
|
|
149
|
-
mesh_config = {
|
|
150
|
-
"system-monitoring-log-apm": {
|
|
151
|
-
"mesh_address": "system-monitoring-log-apm.service.sim.consul",
|
|
152
|
-
"mesh_port": 8200,
|
|
153
|
-
},
|
|
154
|
-
"automated-processing-scratch-inventory": {"mesh_address": "localhost", "mesh_port": 6379},
|
|
155
|
-
"internal-api-gateway": {"mesh_address": "localhost", "mesh_port": 80},
|
|
156
|
-
}
|
|
157
|
-
apm_options = {"TRANSACTION_MAX_SPANS": 10000}
|
|
158
|
-
os.environ["MESH_CONFIG"] = json.dumps(mesh_config)
|
|
159
|
-
os.environ["ELASTIC_APM_ENABLED"] = "true"
|
|
160
|
-
os.environ["ELASTIC_APM_OTHER_OPTIONS"] = json.dumps(apm_options)
|
|
161
|
-
|
|
162
|
-
|
|
163
149
|
def main(
|
|
164
150
|
scratch_path: str,
|
|
165
151
|
suffix: str = "FITS",
|
|
@@ -177,11 +163,8 @@ def main(
|
|
|
177
163
|
load_calibrated_data: bool = False,
|
|
178
164
|
force_intensity_only: bool = False,
|
|
179
165
|
transfer_trial_data: str | None = None,
|
|
180
|
-
use_apm: bool = False,
|
|
181
166
|
dummy_wavelength: float = 630.0,
|
|
182
167
|
):
|
|
183
|
-
if use_apm:
|
|
184
|
-
setup_APM_config()
|
|
185
168
|
with ManualProcessing(
|
|
186
169
|
workflow_path=scratch_path,
|
|
187
170
|
recipe_run_id=recipe_run_id,
|
|
@@ -375,13 +358,12 @@ if __name__ == "__main__":
|
|
|
375
358
|
parser.add_argument(
|
|
376
359
|
"-O",
|
|
377
360
|
"--load-solar-gain-as-science",
|
|
378
|
-
help="Don't re-make the
|
|
361
|
+
help="Don't re-make the solar-gain-as-science frames",
|
|
379
362
|
action="store_true",
|
|
380
363
|
)
|
|
381
364
|
parser.add_argument(
|
|
382
365
|
"-C", "--load-calibrated-data", help="Load CALIBRATED 'science' frames", action="store_true"
|
|
383
366
|
)
|
|
384
|
-
parser.add_argument("-A", "--use-apm", help="Send APM spans to SIM", action="store_true")
|
|
385
367
|
args = parser.parse_args()
|
|
386
368
|
sys.exit(
|
|
387
369
|
main(
|
|
@@ -401,6 +383,5 @@ if __name__ == "__main__":
|
|
|
401
383
|
load_calibrated_data=args.load_calibrated_data,
|
|
402
384
|
force_intensity_only=args.force_I_only,
|
|
403
385
|
transfer_trial_data=args.transfer_trial_data,
|
|
404
|
-
use_apm=args.use_apm,
|
|
405
386
|
)
|
|
406
387
|
)
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import argparse
|
|
2
2
|
import json
|
|
3
|
-
import os
|
|
4
3
|
import sys
|
|
5
4
|
from dataclasses import asdict
|
|
6
5
|
from pathlib import Path
|
|
@@ -91,6 +90,7 @@ class CreateInputDatasetParameterDocument(WorkflowTaskBase):
|
|
|
91
90
|
relative_path=relative_path,
|
|
92
91
|
tags=VispTag.input_dataset_parameters(),
|
|
93
92
|
encoder=basemodel_encoder,
|
|
93
|
+
overwrite=True,
|
|
94
94
|
)
|
|
95
95
|
logger.info(f"Wrote input dataset parameter doc to {relative_path}")
|
|
96
96
|
|
|
@@ -166,21 +166,6 @@ class ValidateL1Output(VispTaskBase):
|
|
|
166
166
|
spec214_validator.validate(f, extra=False)
|
|
167
167
|
|
|
168
168
|
|
|
169
|
-
def setup_APM_config() -> None:
|
|
170
|
-
mesh_config = {
|
|
171
|
-
"system-monitoring-log-apm": {
|
|
172
|
-
"mesh_address": "system-monitoring-log-apm.service.sim.consul",
|
|
173
|
-
"mesh_port": 8200,
|
|
174
|
-
},
|
|
175
|
-
"automated-processing-scratch-inventory": {"mesh_address": "localhost", "mesh_port": 6379},
|
|
176
|
-
"internal-api-gateway": {"mesh_address": "localhost", "mesh_port": 80},
|
|
177
|
-
}
|
|
178
|
-
apm_options = {"TRANSACTION_MAX_SPANS": 10000}
|
|
179
|
-
os.environ["MESH_CONFIG"] = json.dumps(mesh_config)
|
|
180
|
-
os.environ["ELASTIC_APM_ENABLED"] = "true"
|
|
181
|
-
os.environ["ELASTIC_APM_OTHER_OPTIONS"] = json.dumps(apm_options)
|
|
182
|
-
|
|
183
|
-
|
|
184
169
|
def main(
|
|
185
170
|
scratch_path: str,
|
|
186
171
|
suffix: str = "FITS",
|
|
@@ -194,11 +179,8 @@ def main(
|
|
|
194
179
|
load_geometric: bool = False,
|
|
195
180
|
load_solar: bool = False,
|
|
196
181
|
load_inst_pol: bool = False,
|
|
197
|
-
use_apm: bool = False,
|
|
198
182
|
transfer_trial_data: str | None = None,
|
|
199
183
|
):
|
|
200
|
-
if use_apm:
|
|
201
|
-
setup_APM_config()
|
|
202
184
|
with ManualProcessing(
|
|
203
185
|
workflow_path=scratch_path,
|
|
204
186
|
recipe_run_id=recipe_run_id,
|
|
@@ -369,7 +351,6 @@ if __name__ == "__main__":
|
|
|
369
351
|
help="Load instrument polarization calibration from previously saved run",
|
|
370
352
|
action="store_true",
|
|
371
353
|
)
|
|
372
|
-
parser.add_argument("-A", "--use-apm", help="Send APM spans to SIM", action="store_true")
|
|
373
354
|
args = parser.parse_args()
|
|
374
355
|
sys.exit(
|
|
375
356
|
main(
|
|
@@ -385,7 +366,6 @@ if __name__ == "__main__":
|
|
|
385
366
|
load_geometric=args.load_geometric,
|
|
386
367
|
load_solar=args.load_solar,
|
|
387
368
|
load_inst_pol=args.load_inst_pol,
|
|
388
|
-
use_apm=args.use_apm,
|
|
389
369
|
transfer_trial_data=args.transfer_trial_data,
|
|
390
370
|
)
|
|
391
371
|
)
|