dkist-processing-visp 3.3.0__py3-none-any.whl → 5.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. dkist_processing_visp/__init__.py +1 -0
  2. dkist_processing_visp/config.py +1 -0
  3. dkist_processing_visp/models/constants.py +52 -21
  4. dkist_processing_visp/models/fits_access.py +20 -0
  5. dkist_processing_visp/models/metric_code.py +10 -0
  6. dkist_processing_visp/models/parameters.py +129 -19
  7. dkist_processing_visp/models/tags.py +1 -0
  8. dkist_processing_visp/models/task_name.py +1 -0
  9. dkist_processing_visp/parsers/map_repeats.py +1 -0
  10. dkist_processing_visp/parsers/modulator_states.py +1 -0
  11. dkist_processing_visp/parsers/polarimeter_mode.py +3 -1
  12. dkist_processing_visp/parsers/raster_step.py +4 -1
  13. dkist_processing_visp/parsers/spectrograph_configuration.py +75 -0
  14. dkist_processing_visp/parsers/time.py +15 -7
  15. dkist_processing_visp/parsers/visp_l0_fits_access.py +19 -8
  16. dkist_processing_visp/parsers/visp_l1_fits_access.py +1 -0
  17. dkist_processing_visp/tasks/__init__.py +1 -0
  18. dkist_processing_visp/tasks/assemble_movie.py +1 -0
  19. dkist_processing_visp/tasks/background_light.py +2 -1
  20. dkist_processing_visp/tasks/dark.py +5 -4
  21. dkist_processing_visp/tasks/geometric.py +132 -20
  22. dkist_processing_visp/tasks/instrument_polarization.py +13 -12
  23. dkist_processing_visp/tasks/l1_output_data.py +203 -0
  24. dkist_processing_visp/tasks/lamp.py +53 -93
  25. dkist_processing_visp/tasks/make_movie_frames.py +8 -6
  26. dkist_processing_visp/tasks/mixin/beam_access.py +1 -0
  27. dkist_processing_visp/tasks/mixin/corrections.py +54 -4
  28. dkist_processing_visp/tasks/mixin/downsample.py +1 -0
  29. dkist_processing_visp/tasks/parse.py +34 -4
  30. dkist_processing_visp/tasks/quality_metrics.py +5 -4
  31. dkist_processing_visp/tasks/science.py +126 -46
  32. dkist_processing_visp/tasks/solar.py +896 -456
  33. dkist_processing_visp/tasks/visp_base.py +2 -0
  34. dkist_processing_visp/tasks/write_l1.py +25 -5
  35. dkist_processing_visp/tests/conftest.py +99 -35
  36. dkist_processing_visp/tests/header_models.py +92 -20
  37. dkist_processing_visp/tests/local_trial_workflows/l0_cals_only.py +4 -23
  38. dkist_processing_visp/tests/local_trial_workflows/l0_polcals_as_science.py +421 -0
  39. dkist_processing_visp/tests/local_trial_workflows/l0_solar_gain_as_science.py +10 -29
  40. dkist_processing_visp/tests/local_trial_workflows/l0_to_l1.py +1 -21
  41. dkist_processing_visp/tests/local_trial_workflows/local_trial_helpers.py +98 -14
  42. dkist_processing_visp/tests/test_assemble_movie.py +2 -3
  43. dkist_processing_visp/tests/test_assemble_quality.py +89 -4
  44. dkist_processing_visp/tests/test_background_light.py +8 -5
  45. dkist_processing_visp/tests/test_dark.py +4 -3
  46. dkist_processing_visp/tests/test_fits_access.py +43 -0
  47. dkist_processing_visp/tests/test_geometric.py +45 -4
  48. dkist_processing_visp/tests/test_instrument_polarization.py +4 -3
  49. dkist_processing_visp/tests/test_lamp.py +22 -26
  50. dkist_processing_visp/tests/test_make_movie_frames.py +4 -4
  51. dkist_processing_visp/tests/test_map_repeats.py +3 -1
  52. dkist_processing_visp/tests/test_parameters.py +122 -21
  53. dkist_processing_visp/tests/test_parse.py +98 -14
  54. dkist_processing_visp/tests/test_quality.py +2 -3
  55. dkist_processing_visp/tests/test_science.py +113 -15
  56. dkist_processing_visp/tests/test_solar.py +318 -99
  57. dkist_processing_visp/tests/test_visp_constants.py +36 -8
  58. dkist_processing_visp/tests/test_workflows.py +1 -0
  59. dkist_processing_visp/tests/test_write_l1.py +17 -3
  60. dkist_processing_visp/workflows/__init__.py +1 -0
  61. dkist_processing_visp/workflows/l0_processing.py +8 -2
  62. dkist_processing_visp/workflows/trial_workflows.py +8 -2
  63. dkist_processing_visp-5.1.1.dist-info/METADATA +552 -0
  64. dkist_processing_visp-5.1.1.dist-info/RECORD +94 -0
  65. docs/conf.py +5 -1
  66. docs/gain_correction.rst +50 -42
  67. dkist_processing_visp/tasks/mixin/line_zones.py +0 -115
  68. dkist_processing_visp-3.3.0.dist-info/METADATA +0 -459
  69. dkist_processing_visp-3.3.0.dist-info/RECORD +0 -90
  70. {dkist_processing_visp-3.3.0.dist-info → dkist_processing_visp-5.1.1.dist-info}/WHEEL +0 -0
  71. {dkist_processing_visp-3.3.0.dist-info → dkist_processing_visp-5.1.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,421 @@
1
+ """For running just PolCal frames as science data and saving their outputs."""
2
+
3
+ import argparse
4
+ import json
5
+ import logging
6
+ import sys
7
+ from dataclasses import asdict
8
+ from random import randint
9
+ from typing import Literal
10
+
11
+ from astropy.io import fits
12
+ from dkist_processing_common.codecs.basemodel import basemodel_encoder
13
+ from dkist_processing_common.manual import ManualProcessing
14
+ from dkist_processing_common.models.fits_access import MetadataKey
15
+ from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
16
+ from dkist_processing_common.tasks import CreateTrialQualityReport
17
+ from dkist_processing_common.tasks import QualityL1Metrics
18
+ from dkist_processing_common.tasks import WorkflowTaskBase
19
+ from dkist_service_configuration.logging import logger
20
+
21
+ from dkist_processing_visp.models.constants import VispBudName
22
+ from dkist_processing_visp.models.fits_access import VispMetadataKey
23
+ from dkist_processing_visp.models.tags import VispTag
24
+ from dkist_processing_visp.tasks import AssembleVispMovie
25
+ from dkist_processing_visp.tasks import MakeVispMovieFrames
26
+ from dkist_processing_visp.tasks.background_light import BackgroundLightCalibration
27
+ from dkist_processing_visp.tasks.dark import DarkCalibration
28
+ from dkist_processing_visp.tasks.geometric import GeometricCalibration
29
+ from dkist_processing_visp.tasks.instrument_polarization import InstrumentPolarizationCalibration
30
+ from dkist_processing_visp.tasks.l1_output_data import VispAssembleQualityData
31
+ from dkist_processing_visp.tasks.lamp import LampCalibration
32
+ from dkist_processing_visp.tasks.science import ScienceCalibration
33
+ from dkist_processing_visp.tasks.solar import SolarCalibration
34
+ from dkist_processing_visp.tasks.visp_base import VispTaskBase
35
+ from dkist_processing_visp.tasks.write_l1 import VispWriteL1Frame
36
+ from dkist_processing_visp.tests.conftest import VispInputDatasetParameterValues
37
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadBackgroundCal
38
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadCalibratedData
39
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadDarkCal
40
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadGeometricCal
41
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadInputParsing
42
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadInstPolCal
43
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadLampCal
44
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
45
+ LoadPolcalAsScience,
46
+ )
47
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadSolarCal
48
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
49
+ ParseCalOnlyL0InputData,
50
+ )
51
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveBackgroundCal
52
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveCalibratedData
53
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveDarkCal
54
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveGeometricCal
55
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveInputParsing
56
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveInstPolCal
57
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveLampCal
58
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
59
+ SavePolcalAsScience,
60
+ )
61
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveSolarCal
62
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SetAxesTypes
63
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
64
+ SetCadenceConstants,
65
+ )
66
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SetNumModstates
67
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SetObserveExpTime
68
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
69
+ SetObserveIpStartTime,
70
+ )
71
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SetPolarimeterMode
72
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import ValidateL1Output
73
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
74
+ set_observe_wavelength_task,
75
+ )
76
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import tag_inputs_task
77
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
78
+ translate_122_to_214l0_task,
79
+ )
80
+
81
+ __version__ = "PCAS"
82
+
83
+ QUALITY = False
84
+ try:
85
+ import dkist_quality
86
+
87
+ QUALITY = True
88
+ except ModuleNotFoundError:
89
+ logging.warning("Could not find dkist-quality")
90
+
91
+
92
+ class CreateInputDatasetParameterDocument(WorkflowTaskBase):
93
+ def run(self) -> None:
94
+ relative_path = "input_dataset_parameters.json"
95
+ self.write(
96
+ data=InputDatasetPartDocumentList(
97
+ doc_list=self.input_dataset_document_simple_parameters_part
98
+ ),
99
+ relative_path=relative_path,
100
+ tags=VispTag.input_dataset_parameters(),
101
+ encoder=basemodel_encoder,
102
+ overwrite=True,
103
+ )
104
+ logger.info(f"Wrote input dataset parameter doc to {relative_path}")
105
+
106
+ @property
107
+ def input_dataset_document_simple_parameters_part(self):
108
+ parameters_list = []
109
+ value_id = randint(1000, 2000)
110
+ for pn, pv in asdict(
111
+ VispInputDatasetParameterValues(
112
+ visp_background_on=False,
113
+ visp_geo_upsample_factor=10000,
114
+ visp_polcal_num_spatial_bins=2560,
115
+ )
116
+ ).items():
117
+ values = [
118
+ {
119
+ "parameterValueId": value_id,
120
+ "parameterValue": json.dumps(pv),
121
+ "parameterValueStartDate": "1946-11-20",
122
+ }
123
+ ]
124
+ parameter = {"parameterName": pn, "parameterValues": values}
125
+ parameters_list.append(parameter)
126
+
127
+ return parameters_list
128
+
129
+
130
+ class TagPolcalAsScience(VispTaskBase):
131
+ """Do."""
132
+
133
+ def run(self) -> None:
134
+ """Do."""
135
+ # First, tag the polcal frames as observe frames with the correct stuff
136
+ num_raster_set = set()
137
+ for cs_step in range(self.constants.num_cs_steps):
138
+ for modstate in range(1, self.constants.num_modstates + 1):
139
+ file_list = list(
140
+ self.read(
141
+ tags=[
142
+ VispTag.task_polcal(),
143
+ VispTag.cs_step(cs_step),
144
+ VispTag.modstate(modstate),
145
+ ]
146
+ )
147
+ )
148
+ num_raster_set.add(len(file_list))
149
+ for raster_step, file_name in enumerate(file_list):
150
+ og_tags = self.tags(file_name)
151
+ logging.info(
152
+ f"Raw frame {file_name} at {cs_step = } and {modstate = } has {og_tags = }"
153
+ )
154
+ self.scratch._tag_db.clear_value(file_name)
155
+ logging.info(f"\tafter removing, the tags are {self.tags(file_name)}")
156
+
157
+ hdul = fits.open(file_name, mode="update")
158
+ idx = 0
159
+ if hdul[idx].data is None:
160
+ idx = 1
161
+ hdul[idx].header[VispMetadataKey.raster_scan_step] = raster_step
162
+ hdul.flush()
163
+ del hdul
164
+
165
+ new_tags = [
166
+ VispTag.task_observe(),
167
+ VispTag.map_scan(cs_step + 1),
168
+ VispTag.raster_step(raster_step),
169
+ VispTag.frame(),
170
+ VispTag.modstate(modstate),
171
+ VispTag.input(),
172
+ VispTag.readout_exp_time(self.constants.observe_readout_exp_times[0]),
173
+ ]
174
+ logging.info(f"\tadding {new_tags = }")
175
+ self.tag(file_name, new_tags)
176
+ final_tags = self.tags(file_name)
177
+ logging.info(f"\tafter retagging tags are {final_tags}")
178
+
179
+ if len(num_raster_set) != 1:
180
+ raise ValueError(
181
+ "Expected to find the same number of files for each (CS step, modstate), but did not. "
182
+ f"Set of counts is {num_raster_set}"
183
+ )
184
+
185
+ num_raster_steps = num_raster_set.pop()
186
+ logging.info(f"Found {num_raster_steps} raster steps")
187
+ # Now update the num[map, raster] constants. We'll call each CS step a "map"
188
+ self.constants._update(
189
+ {
190
+ VispBudName.num_raster_steps.value: num_raster_steps,
191
+ VispBudName.num_map_scans.value: self.constants.num_cs_steps,
192
+ }
193
+ )
194
+
195
+
196
+ def write_L1_files_task(prefix: str = ""):
197
+ """Do."""
198
+ if len(prefix) > 0 and prefix[-1] != "_":
199
+ prefix += "_"
200
+
201
+ class WritePolcalL1Files(VispWriteL1Frame):
202
+ """Do."""
203
+
204
+ def l1_filename(self, header: fits.Header, stokes: Literal["I", "Q", "U", "V"]):
205
+ """Do."""
206
+ wavelength = str(round(header[MetadataKey.wavelength] * 1000)).zfill(8)
207
+ cs_step = header["VSPMAP"]
208
+ raster_step = header[VispMetadataKey.raster_scan_step]
209
+ return f"{prefix}CS_STEP_{cs_step:02n}_{raster_step:02n}_{wavelength}_{stokes}_L1.fits"
210
+
211
+ return WritePolcalL1Files
212
+
213
+
214
+ def main(
215
+ scratch_path: str,
216
+ suffix: str = "fits",
217
+ prefix: str = "",
218
+ recipe_run_id: int = 2,
219
+ skip_translation: bool = False,
220
+ only_translate: bool = False,
221
+ load_parse: bool = False,
222
+ load_dark: bool = False,
223
+ load_background: bool = False,
224
+ load_lamp: bool = False,
225
+ load_geometric: bool = False,
226
+ load_solar: bool = False,
227
+ load_inst_pol: bool = False,
228
+ load_polcal_as_science: bool = False,
229
+ load_calibrated_data: bool = False,
230
+ dummy_wavelength: float = 630.0,
231
+ ):
232
+ """Run the damn thing."""
233
+ with ManualProcessing(
234
+ workflow_path=scratch_path,
235
+ recipe_run_id=recipe_run_id,
236
+ testing=True,
237
+ workflow_name="visp-l0-pipeline",
238
+ workflow_version="GROGU",
239
+ ) as manual_processing_run:
240
+ if not skip_translation:
241
+ manual_processing_run.run_task(task=translate_122_to_214l0_task(suffix))
242
+ if only_translate:
243
+ return
244
+ manual_processing_run.run_task(task=CreateInputDatasetParameterDocument)
245
+ if load_parse:
246
+ manual_processing_run.run_task(task=LoadInputParsing)
247
+ else:
248
+ manual_processing_run.run_task(task=tag_inputs_task(suffix))
249
+ manual_processing_run.run_task(task=ParseCalOnlyL0InputData)
250
+ manual_processing_run.run_task(
251
+ task=set_observe_wavelength_task(wavelength=dummy_wavelength)
252
+ )
253
+ manual_processing_run.run_task(task=SetObserveIpStartTime)
254
+ manual_processing_run.run_task(task=SetObserveExpTime)
255
+ manual_processing_run.run_task(task=SetPolarimeterMode)
256
+ manual_processing_run.run_task(task=SetNumModstates)
257
+ manual_processing_run.run_task(task=SetCadenceConstants)
258
+ manual_processing_run.run_task(task=SetAxesTypes)
259
+ manual_processing_run.run_task(task=SaveInputParsing)
260
+
261
+ if load_dark:
262
+ manual_processing_run.run_task(task=LoadDarkCal)
263
+ else:
264
+ manual_processing_run.run_task(task=DarkCalibration)
265
+ manual_processing_run.run_task(task=SaveDarkCal)
266
+
267
+ if load_background:
268
+ manual_processing_run.run_task(task=LoadBackgroundCal)
269
+ else:
270
+ manual_processing_run.run_task(task=BackgroundLightCalibration)
271
+ manual_processing_run.run_task(task=SaveBackgroundCal)
272
+
273
+ if load_lamp:
274
+ manual_processing_run.run_task(task=LoadLampCal)
275
+ else:
276
+ manual_processing_run.run_task(task=LampCalibration)
277
+ manual_processing_run.run_task(task=SaveLampCal)
278
+
279
+ if load_geometric:
280
+ manual_processing_run.run_task(task=LoadGeometricCal)
281
+ else:
282
+ manual_processing_run.run_task(task=GeometricCalibration)
283
+ manual_processing_run.run_task(task=SaveGeometricCal)
284
+
285
+ if load_solar:
286
+ manual_processing_run.run_task(task=LoadSolarCal)
287
+ else:
288
+ manual_processing_run.run_task(task=SolarCalibration)
289
+ manual_processing_run.run_task(task=SaveSolarCal)
290
+
291
+ if load_inst_pol:
292
+ manual_processing_run.run_task(task=LoadInstPolCal)
293
+ else:
294
+ manual_processing_run.run_task(task=InstrumentPolarizationCalibration)
295
+ manual_processing_run.run_task(task=SaveInstPolCal)
296
+
297
+ if load_polcal_as_science:
298
+ manual_processing_run.run_task(task=LoadPolcalAsScience)
299
+ else:
300
+ manual_processing_run.run_task(task=TagPolcalAsScience)
301
+ manual_processing_run.run_task(task=SavePolcalAsScience)
302
+
303
+ if load_calibrated_data:
304
+ manual_processing_run.run_task(task=LoadCalibratedData)
305
+ else:
306
+ manual_processing_run.run_task(task=ScienceCalibration)
307
+ manual_processing_run.run_task(task=SaveCalibratedData)
308
+
309
+ manual_processing_run.run_task(task=write_L1_files_task(prefix=prefix))
310
+ manual_processing_run.run_task(task=QualityL1Metrics)
311
+ manual_processing_run.run_task(task=VispAssembleQualityData)
312
+ manual_processing_run.run_task(task=ValidateL1Output)
313
+ manual_processing_run.run_task(task=MakeVispMovieFrames)
314
+ manual_processing_run.run_task(task=AssembleVispMovie)
315
+
316
+ # Test some downstream services
317
+ if QUALITY:
318
+ manual_processing_run.run_task(task=CreateTrialQualityReport)
319
+ else:
320
+ logger.warning("Did NOT make quality report pdf because dkist-quality is not installed")
321
+
322
+
323
+ if __name__ == "__main__":
324
+ parser = argparse.ArgumentParser(
325
+ description="Generate PolCals with the ViSP DC Science pipeline"
326
+ )
327
+ parser.add_argument("scratch_path", help="Location to use as the DC 'scratch' disk")
328
+ parser.add_argument(
329
+ "-i",
330
+ "--run-id",
331
+ help="Which subdir to use. This will become the recipe run id",
332
+ type=int,
333
+ )
334
+ parser.add_argument("--prefix", help="File prefix to add to L1 output frames", default="")
335
+ parser.add_argument("--suffix", help="File suffix to treat as INPUT frames", default="fits")
336
+ parser.add_argument(
337
+ "-w",
338
+ "--wavelength",
339
+ help="Dummy wavelength to use for loading parameters, etc.",
340
+ type=float,
341
+ default=630.0,
342
+ )
343
+ parser.add_argument(
344
+ "-T",
345
+ "--skip-translation",
346
+ help="Skip the translation of raw 122 l0 frames to 214 l0",
347
+ action="store_true",
348
+ )
349
+ parser.add_argument(
350
+ "-t", "--only-translate", help="Do ONLY the translation step", action="store_true"
351
+ )
352
+ parser.add_argument(
353
+ "-I", "--load-input-parsing", help="Load tags on input files", action="store_true"
354
+ )
355
+ parser.add_argument(
356
+ "-D",
357
+ "--load-dark",
358
+ help="Load dark calibration from previously saved run",
359
+ action="store_true",
360
+ )
361
+ parser.add_argument(
362
+ "-B",
363
+ "--load-background",
364
+ help="Load background calibration from previously saved run",
365
+ action="store_true",
366
+ )
367
+ parser.add_argument(
368
+ "-L",
369
+ "--load-lamp",
370
+ help="Load lamp calibration from previously saved run",
371
+ action="store_true",
372
+ )
373
+ parser.add_argument(
374
+ "-G",
375
+ "--load-geometric",
376
+ help="Load geometric calibration from previously saved run",
377
+ action="store_true",
378
+ )
379
+ parser.add_argument(
380
+ "-S",
381
+ "--load-solar",
382
+ help="Load solar calibration from previously saved run",
383
+ action="store_true",
384
+ )
385
+ parser.add_argument(
386
+ "-P",
387
+ "--load-inst-pol",
388
+ help="Load instrument polarization calibration from previously saved run",
389
+ action="store_true",
390
+ )
391
+ parser.add_argument(
392
+ "-O",
393
+ "--load-polcal-as-science",
394
+ help="Don't re-make the polcal-as-science frames",
395
+ action="store_true",
396
+ )
397
+ parser.add_argument(
398
+ "-C", "--load-calibrated-data", help="Load CALIBRATED 'science' frames", action="store_true"
399
+ )
400
+ args = parser.parse_args()
401
+ logging.info(f"Called as {' '.join(sys.argv)}")
402
+ sys.exit(
403
+ main(
404
+ scratch_path=args.scratch_path,
405
+ suffix=args.suffix,
406
+ prefix=args.prefix,
407
+ recipe_run_id=args.run_id,
408
+ skip_translation=args.skip_translation,
409
+ only_translate=args.only_translate,
410
+ load_parse=args.load_input_parsing,
411
+ load_dark=args.load_dark,
412
+ load_background=args.load_background,
413
+ load_lamp=args.load_lamp,
414
+ load_geometric=args.load_geometric,
415
+ load_solar=args.load_solar,
416
+ load_inst_pol=args.load_inst_pol,
417
+ load_polcal_as_science=args.load_polcal_as_science,
418
+ load_calibrated_data=args.load_calibrated_data,
419
+ dummy_wavelength=args.wavelength,
420
+ )
421
+ )
@@ -29,9 +29,6 @@ from dkist_processing_visp.tasks.instrument_polarization import InstrumentPolari
29
29
  from dkist_processing_visp.tasks.lamp import LampCalibration
30
30
  from dkist_processing_visp.tasks.solar import SolarCalibration
31
31
  from dkist_processing_visp.tests.conftest import VispInputDatasetParameterValues
32
- from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
33
- load_solar_gain_as_science_task,
34
- )
35
32
  from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadBackgroundCal
36
33
  from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadCalibratedData
37
34
  from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import LoadDarkCal
@@ -54,9 +51,6 @@ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers impor
54
51
  from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
55
52
  SaveSolarGainAsScience,
56
53
  )
57
- from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
58
- set_observe_wavelength_task,
59
- )
60
54
  from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SetAxesTypes
61
55
  from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
62
56
  SetCadenceConstants,
@@ -67,20 +61,26 @@ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers impor
67
61
  SetObserveIpStartTime,
68
62
  )
69
63
  from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SetPolarimeterMode
70
- from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import tag_inputs_task
71
64
  from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
72
65
  TagModulatedSolarGainsAsScience,
73
66
  )
74
67
  from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
75
68
  TagSingleSolarGainAsScience,
76
69
  )
70
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import ValidateL1Output
71
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
72
+ load_solar_gain_as_science_task,
73
+ )
74
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
75
+ set_observe_wavelength_task,
76
+ )
77
+ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import tag_inputs_task
77
78
  from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
78
79
  transfer_trial_data_locally_task,
79
80
  )
80
81
  from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
81
82
  translate_122_to_214l0_task,
82
83
  )
83
- from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import ValidateL1Output
84
84
 
85
85
  QUALITY = False
86
86
  try:
@@ -118,6 +118,7 @@ class CreateInputDatasetParameterDocument(WorkflowTaskBase):
118
118
  relative_path=relative_path,
119
119
  tags=VispTag.input_dataset_parameters(),
120
120
  encoder=basemodel_encoder,
121
+ overwrite=True,
121
122
  )
122
123
  logger.info(f"Wrote input dataset parameter doc to {relative_path}")
123
124
 
@@ -145,21 +146,6 @@ class CreateInputDatasetParameterDocument(WorkflowTaskBase):
145
146
  return parameters_list
146
147
 
147
148
 
148
- def setup_APM_config() -> None:
149
- mesh_config = {
150
- "system-monitoring-log-apm": {
151
- "mesh_address": "system-monitoring-log-apm.service.sim.consul",
152
- "mesh_port": 8200,
153
- },
154
- "automated-processing-scratch-inventory": {"mesh_address": "localhost", "mesh_port": 6379},
155
- "internal-api-gateway": {"mesh_address": "localhost", "mesh_port": 80},
156
- }
157
- apm_options = {"TRANSACTION_MAX_SPANS": 10000}
158
- os.environ["MESH_CONFIG"] = json.dumps(mesh_config)
159
- os.environ["ELASTIC_APM_ENABLED"] = "true"
160
- os.environ["ELASTIC_APM_OTHER_OPTIONS"] = json.dumps(apm_options)
161
-
162
-
163
149
  def main(
164
150
  scratch_path: str,
165
151
  suffix: str = "FITS",
@@ -177,11 +163,8 @@ def main(
177
163
  load_calibrated_data: bool = False,
178
164
  force_intensity_only: bool = False,
179
165
  transfer_trial_data: str | None = None,
180
- use_apm: bool = False,
181
166
  dummy_wavelength: float = 630.0,
182
167
  ):
183
- if use_apm:
184
- setup_APM_config()
185
168
  with ManualProcessing(
186
169
  workflow_path=scratch_path,
187
170
  recipe_run_id=recipe_run_id,
@@ -375,13 +358,12 @@ if __name__ == "__main__":
375
358
  parser.add_argument(
376
359
  "-O",
377
360
  "--load-solar-gain-as-science",
378
- help="Don't re-make the polcals-as-science frames",
361
+ help="Don't re-make the solar-gain-as-science frames",
379
362
  action="store_true",
380
363
  )
381
364
  parser.add_argument(
382
365
  "-C", "--load-calibrated-data", help="Load CALIBRATED 'science' frames", action="store_true"
383
366
  )
384
- parser.add_argument("-A", "--use-apm", help="Send APM spans to SIM", action="store_true")
385
367
  args = parser.parse_args()
386
368
  sys.exit(
387
369
  main(
@@ -401,6 +383,5 @@ if __name__ == "__main__":
401
383
  load_calibrated_data=args.load_calibrated_data,
402
384
  force_intensity_only=args.force_I_only,
403
385
  transfer_trial_data=args.transfer_trial_data,
404
- use_apm=args.use_apm,
405
386
  )
406
387
  )
@@ -1,6 +1,5 @@
1
1
  import argparse
2
2
  import json
3
- import os
4
3
  import sys
5
4
  from dataclasses import asdict
6
5
  from pathlib import Path
@@ -91,6 +90,7 @@ class CreateInputDatasetParameterDocument(WorkflowTaskBase):
91
90
  relative_path=relative_path,
92
91
  tags=VispTag.input_dataset_parameters(),
93
92
  encoder=basemodel_encoder,
93
+ overwrite=True,
94
94
  )
95
95
  logger.info(f"Wrote input dataset parameter doc to {relative_path}")
96
96
 
@@ -166,21 +166,6 @@ class ValidateL1Output(VispTaskBase):
166
166
  spec214_validator.validate(f, extra=False)
167
167
 
168
168
 
169
- def setup_APM_config() -> None:
170
- mesh_config = {
171
- "system-monitoring-log-apm": {
172
- "mesh_address": "system-monitoring-log-apm.service.sim.consul",
173
- "mesh_port": 8200,
174
- },
175
- "automated-processing-scratch-inventory": {"mesh_address": "localhost", "mesh_port": 6379},
176
- "internal-api-gateway": {"mesh_address": "localhost", "mesh_port": 80},
177
- }
178
- apm_options = {"TRANSACTION_MAX_SPANS": 10000}
179
- os.environ["MESH_CONFIG"] = json.dumps(mesh_config)
180
- os.environ["ELASTIC_APM_ENABLED"] = "true"
181
- os.environ["ELASTIC_APM_OTHER_OPTIONS"] = json.dumps(apm_options)
182
-
183
-
184
169
  def main(
185
170
  scratch_path: str,
186
171
  suffix: str = "FITS",
@@ -194,11 +179,8 @@ def main(
194
179
  load_geometric: bool = False,
195
180
  load_solar: bool = False,
196
181
  load_inst_pol: bool = False,
197
- use_apm: bool = False,
198
182
  transfer_trial_data: str | None = None,
199
183
  ):
200
- if use_apm:
201
- setup_APM_config()
202
184
  with ManualProcessing(
203
185
  workflow_path=scratch_path,
204
186
  recipe_run_id=recipe_run_id,
@@ -369,7 +351,6 @@ if __name__ == "__main__":
369
351
  help="Load instrument polarization calibration from previously saved run",
370
352
  action="store_true",
371
353
  )
372
- parser.add_argument("-A", "--use-apm", help="Send APM spans to SIM", action="store_true")
373
354
  args = parser.parse_args()
374
355
  sys.exit(
375
356
  main(
@@ -385,7 +366,6 @@ if __name__ == "__main__":
385
366
  load_geometric=args.load_geometric,
386
367
  load_solar=args.load_solar,
387
368
  load_inst_pol=args.load_inst_pol,
388
- use_apm=args.use_apm,
389
369
  transfer_trial_data=args.transfer_trial_data,
390
370
  )
391
371
  )