dkist-processing-visp 3.6.3__py3-none-any.whl → 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dkist_processing_visp/tasks/make_movie_frames.py +1 -1
- dkist_processing_visp/tasks/mixin/corrections.py +53 -4
- dkist_processing_visp/tasks/quality_metrics.py +1 -1
- dkist_processing_visp/tasks/science.py +99 -22
- dkist_processing_visp/tests/test_science.py +103 -8
- {dkist_processing_visp-3.6.3.dist-info → dkist_processing_visp-4.0.0.dist-info}/METADATA +13 -14
- {dkist_processing_visp-3.6.3.dist-info → dkist_processing_visp-4.0.0.dist-info}/RECORD +9 -9
- {dkist_processing_visp-3.6.3.dist-info → dkist_processing_visp-4.0.0.dist-info}/WHEEL +0 -0
- {dkist_processing_visp-3.6.3.dist-info → dkist_processing_visp-4.0.0.dist-info}/top_level.txt +0 -0
|
@@ -79,7 +79,7 @@ class MakeVispMovieFrames(VispTaskBase):
|
|
|
79
79
|
fits_access_class=VispL1FitsAccess,
|
|
80
80
|
)
|
|
81
81
|
)
|
|
82
|
-
data = calibrated_frame.data
|
|
82
|
+
data = np.nan_to_num(calibrated_frame.data, nan=0)
|
|
83
83
|
if self.constants.num_raster_steps == 1:
|
|
84
84
|
logger.info(
|
|
85
85
|
"Only a single raster step found. Making a spectral movie."
|
|
@@ -17,6 +17,9 @@ class CorrectionsMixin:
|
|
|
17
17
|
arrays: Iterable[np.ndarray] | np.ndarray,
|
|
18
18
|
shift: np.ndarray = np.zeros(2),
|
|
19
19
|
angle: float = 0.0,
|
|
20
|
+
mode: str = "edge",
|
|
21
|
+
order: int = 5,
|
|
22
|
+
cval: float = np.nan,
|
|
20
23
|
) -> Generator[np.ndarray, None, None]:
|
|
21
24
|
"""
|
|
22
25
|
Shift and then rotate data.
|
|
@@ -35,6 +38,24 @@ class CorrectionsMixin:
|
|
|
35
38
|
angle : float
|
|
36
39
|
The angle (in radians) between slit hairlines and pixel axes.
|
|
37
40
|
|
|
41
|
+
mode : {'constant', 'edge', 'symmetric', 'reflect', 'wrap'}
|
|
42
|
+
Points outside the boundaries of the input are filled according
|
|
43
|
+
to the given mode. Modes match the behaviour of `numpy.pad`.
|
|
44
|
+
|
|
45
|
+
order : int
|
|
46
|
+
The order of interpolation. The order has to be in the range 0-5:
|
|
47
|
+
- 0: Nearest-neighbor
|
|
48
|
+
- 1: Bi-linear (default)
|
|
49
|
+
- 2: Bi-quadratic
|
|
50
|
+
- 3: Bi-cubic
|
|
51
|
+
- 4: Bi-quartic
|
|
52
|
+
- 5: Bi-quintic
|
|
53
|
+
|
|
54
|
+
cval : float
|
|
55
|
+
Used in conjunction with mode 'constant', the value outside
|
|
56
|
+
the image boundaries.
|
|
57
|
+
|
|
58
|
+
|
|
38
59
|
Returns
|
|
39
60
|
-------
|
|
40
61
|
Generator
|
|
@@ -46,12 +67,21 @@ class CorrectionsMixin:
|
|
|
46
67
|
array[np.where(array == np.inf)] = np.max(array[np.isfinite(array)])
|
|
47
68
|
array[np.where(array == -np.inf)] = np.min(array[np.isfinite(array)])
|
|
48
69
|
array[np.isnan(array)] = np.nanmedian(array)
|
|
49
|
-
translated = affine_transform_arrays(
|
|
50
|
-
|
|
70
|
+
translated = affine_transform_arrays(
|
|
71
|
+
array, translation=-shift, mode=mode, order=order, cval=cval
|
|
72
|
+
)
|
|
73
|
+
yield next(
|
|
74
|
+
rotate_arrays_about_point(
|
|
75
|
+
translated, angle=-angle, mode=mode, order=order, cval=cval
|
|
76
|
+
)
|
|
77
|
+
)
|
|
51
78
|
|
|
52
79
|
@staticmethod
|
|
53
80
|
def corrections_remove_spec_geometry(
|
|
54
|
-
arrays: Iterable[np.ndarray] | np.ndarray,
|
|
81
|
+
arrays: Iterable[np.ndarray] | np.ndarray,
|
|
82
|
+
spec_shift: np.ndarray,
|
|
83
|
+
cval: float | None = None,
|
|
84
|
+
order: int = 3,
|
|
55
85
|
) -> Generator[np.ndarray, None, None]:
|
|
56
86
|
"""
|
|
57
87
|
Remove spectral curvature.
|
|
@@ -67,6 +97,19 @@ class CorrectionsMixin:
|
|
|
67
97
|
Array with shape (X), where X is the number of pixels in the spatial dimension.
|
|
68
98
|
This dimension gives the spectral shift.
|
|
69
99
|
|
|
100
|
+
order : int
|
|
101
|
+
The order of interpolation. The order has to be in the range 0-5:
|
|
102
|
+
- 0: Nearest-neighbor
|
|
103
|
+
- 1: Bi-linear (default)
|
|
104
|
+
- 2: Bi-quadratic
|
|
105
|
+
- 3: Bi-cubic
|
|
106
|
+
- 4: Bi-quartic
|
|
107
|
+
- 5: Bi-quintic
|
|
108
|
+
|
|
109
|
+
cval : float
|
|
110
|
+
Used in conjunction with mode 'constant', the value outside
|
|
111
|
+
the image boundaries.
|
|
112
|
+
|
|
70
113
|
Returns
|
|
71
114
|
-------
|
|
72
115
|
Generator
|
|
@@ -78,8 +121,14 @@ class CorrectionsMixin:
|
|
|
78
121
|
numy = array.shape[1]
|
|
79
122
|
array_output = np.zeros(array.shape)
|
|
80
123
|
for j in range(numy):
|
|
124
|
+
if cval is None:
|
|
125
|
+
cval = np.nanmedian(array[:, j])
|
|
81
126
|
array_output[:, j] = spnd.shift(
|
|
82
|
-
array[:, j],
|
|
127
|
+
array[:, j],
|
|
128
|
+
-spec_shift[j],
|
|
129
|
+
mode="constant",
|
|
130
|
+
cval=cval,
|
|
131
|
+
order=order,
|
|
83
132
|
)
|
|
84
133
|
yield array_output
|
|
85
134
|
|
|
@@ -162,7 +162,7 @@ class VispL1QualityMetrics(VispTaskBase, QualityMixin):
|
|
|
162
162
|
continue
|
|
163
163
|
|
|
164
164
|
# compute sensitivity for this Stokes parameter
|
|
165
|
-
data_list.append(np.
|
|
165
|
+
data_list.append(np.nanstd(stokes_frame.data) / stokesI_med)
|
|
166
166
|
|
|
167
167
|
all_datetimes.append(Time(np.mean(polarization_data.datetimes), format="mjd").isot)
|
|
168
168
|
for target, source in zip(
|
|
@@ -51,6 +51,9 @@ class CalibrationCollection:
|
|
|
51
51
|
|
|
52
52
|
This is done by considering that state offset values computed by the GeometricCalibration task. Any sub-pixel
|
|
53
53
|
overlaps are rounded to the next integer that still guarantees overlap.
|
|
54
|
+
|
|
55
|
+
When "start pixels" are mentioned, those are pixels being counted from zero on a given axis in the positive direction.
|
|
56
|
+
When "end pixels" are mentioned, those are pixels being counted from the end of a given axis in the negative direction.
|
|
54
57
|
"""
|
|
55
58
|
logger.info("Computing beam overlap slices")
|
|
56
59
|
# This will be a flat list of (x, y) pairs for all modstates and beams
|
|
@@ -63,25 +66,33 @@ class CalibrationCollection:
|
|
|
63
66
|
logger.info(f"All x shifts: {all_x_shifts}")
|
|
64
67
|
logger.info(f"All y shifts: {all_y_shifts}")
|
|
65
68
|
|
|
66
|
-
# The amount we need to "slice in" from the
|
|
69
|
+
# The amount we need to "slice in" from the start of the array is equivalent to the absolute value of the most negative shift.
|
|
67
70
|
# The call to `np.ceil` ensures that the integer rounding doesn't allow non-overlap regions to leak in.
|
|
68
|
-
|
|
69
|
-
|
|
71
|
+
start_pixels_to_slice_x = int(np.ceil(abs(np.min(all_x_shifts))))
|
|
72
|
+
start_pixels_to_slice_y = int(np.ceil(abs(np.min(all_y_shifts))))
|
|
70
73
|
|
|
71
|
-
# The amount we need to "chop off" the end of the array is the most
|
|
74
|
+
# The amount we need to "chop off" the end of the array is the most positive shift.
|
|
72
75
|
#
|
|
73
76
|
# Here we rely on the fact that the fiducial array's shift is *always* (0, 0)
|
|
74
|
-
#
|
|
75
|
-
#
|
|
76
|
-
#
|
|
77
|
+
# (see `geometric.compute_modstate_offset`). Thus, if there are no negative shifts then the following lines
|
|
78
|
+
# will result in None. This is required for slicing because array[x:0] is no good. So if the max is 0 then we
|
|
79
|
+
# end up with array[x:None] which goes all the way to the end of the array.
|
|
77
80
|
#
|
|
78
|
-
# The call to `np.
|
|
79
|
-
#
|
|
80
|
-
|
|
81
|
-
|
|
81
|
+
# The call to `np.ceil` ensures that the integer rounding doesn't allow non-overlap regions to leak in.
|
|
82
|
+
# (because more negative slices will cut out more data).
|
|
83
|
+
end_pixels_to_slice_x = int(np.ceil(np.max(all_x_shifts))) or None
|
|
84
|
+
end_pixels_to_slice_y = int(np.ceil(np.max(all_y_shifts))) or None
|
|
85
|
+
|
|
86
|
+
# As the pixels to remove from the end of axes is given as a positive number, we need to make it negative for slicing.
|
|
87
|
+
if end_pixels_to_slice_x is not None:
|
|
88
|
+
end_pixels_to_slice_x *= -1
|
|
82
89
|
|
|
83
|
-
|
|
84
|
-
|
|
90
|
+
if end_pixels_to_slice_y is not None:
|
|
91
|
+
end_pixels_to_slice_y *= -1
|
|
92
|
+
|
|
93
|
+
# Construct the slices
|
|
94
|
+
x_slice = slice(start_pixels_to_slice_x, end_pixels_to_slice_x)
|
|
95
|
+
y_slice = slice(start_pixels_to_slice_y, end_pixels_to_slice_y)
|
|
85
96
|
|
|
86
97
|
return x_slice, y_slice
|
|
87
98
|
|
|
@@ -277,6 +288,7 @@ class ScienceCalibration(
|
|
|
277
288
|
for raster_step in range(0, self.constants.num_raster_steps):
|
|
278
289
|
beam_storage = dict()
|
|
279
290
|
header_storage = dict()
|
|
291
|
+
nan_storage = dict()
|
|
280
292
|
for beam in range(1, self.constants.num_beams + 1):
|
|
281
293
|
apm_str = f"{map_scan = }, {raster_step = }, and {beam = }"
|
|
282
294
|
with self.telemetry_span(f"Basic corrections for {apm_str}"):
|
|
@@ -288,6 +300,7 @@ class ScienceCalibration(
|
|
|
288
300
|
(
|
|
289
301
|
intermediate_array,
|
|
290
302
|
intermediate_header,
|
|
303
|
+
nan_mask,
|
|
291
304
|
) = self.process_polarimetric_modstates(
|
|
292
305
|
beam=beam,
|
|
293
306
|
raster_step=raster_step,
|
|
@@ -299,7 +312,11 @@ class ScienceCalibration(
|
|
|
299
312
|
logger.info(
|
|
300
313
|
f"Processing spectrographic observe frames from {apm_str}"
|
|
301
314
|
)
|
|
302
|
-
|
|
315
|
+
(
|
|
316
|
+
intermediate_array,
|
|
317
|
+
intermediate_header,
|
|
318
|
+
nan_mask,
|
|
319
|
+
) = self.correct_single_frame(
|
|
303
320
|
beam=beam,
|
|
304
321
|
modstate=1,
|
|
305
322
|
raster_step=raster_step,
|
|
@@ -310,6 +327,7 @@ class ScienceCalibration(
|
|
|
310
327
|
intermediate_header = self.compute_date_keys(intermediate_header)
|
|
311
328
|
beam_storage[VispTag.beam(beam)] = intermediate_array
|
|
312
329
|
header_storage[VispTag.beam(beam)] = intermediate_header
|
|
330
|
+
nan_storage[VispTag.beam(beam)] = nan_mask
|
|
313
331
|
|
|
314
332
|
with self.telemetry_span("Combining beams"):
|
|
315
333
|
calibrated = self.combine_beams(beam_storage, header_storage, calibrations)
|
|
@@ -318,12 +336,29 @@ class ScienceCalibration(
|
|
|
318
336
|
with self.telemetry_span("Correcting telescope polarization"):
|
|
319
337
|
calibrated = self.telescope_polarization_correction(calibrated)
|
|
320
338
|
|
|
339
|
+
with self.telemetry_span("Combining NaN masks from beams"):
|
|
340
|
+
cut_combined_nan_mask = self.combine_and_cut_nan_masks(
|
|
341
|
+
list(nan_storage.values()), calibrations
|
|
342
|
+
)
|
|
343
|
+
|
|
321
344
|
# Save the final output files
|
|
322
345
|
with self.telemetry_span("Writing calibrated arrays"):
|
|
323
346
|
self.write_calibrated_array(
|
|
324
|
-
calibrated,
|
|
347
|
+
calibrated,
|
|
348
|
+
map_scan=map_scan,
|
|
349
|
+
calibrations=calibrations,
|
|
350
|
+
nan_mask=cut_combined_nan_mask,
|
|
325
351
|
)
|
|
326
352
|
|
|
353
|
+
@staticmethod
|
|
354
|
+
def combine_and_cut_nan_masks(
|
|
355
|
+
nan_masks: list[np.ndarray], calibrations: CalibrationCollection
|
|
356
|
+
) -> np.ndarray:
|
|
357
|
+
"""Combine two NaN masks into one, cropping the result based on pre-calculated shifts."""
|
|
358
|
+
combined_nan_mask = np.logical_or.reduce(nan_masks)
|
|
359
|
+
x_slice, y_slice = calibrations.beams_overlap_slice
|
|
360
|
+
return combined_nan_mask[x_slice, y_slice]
|
|
361
|
+
|
|
327
362
|
def process_polarimetric_modstates(
|
|
328
363
|
self,
|
|
329
364
|
beam: int,
|
|
@@ -331,7 +366,7 @@ class ScienceCalibration(
|
|
|
331
366
|
map_scan: int,
|
|
332
367
|
readout_exp_time: float,
|
|
333
368
|
calibrations: CalibrationCollection,
|
|
334
|
-
) -> tuple[np.ndarray, fits.Header]:
|
|
369
|
+
) -> tuple[np.ndarray, fits.Header, np.ndarray]:
|
|
335
370
|
"""
|
|
336
371
|
Process a single polarimetric beam as much as is possible.
|
|
337
372
|
|
|
@@ -343,11 +378,12 @@ class ScienceCalibration(
|
|
|
343
378
|
].shape
|
|
344
379
|
array_stack = np.zeros(array_shape + (self.constants.num_modstates,))
|
|
345
380
|
header_stack = []
|
|
381
|
+
nan_mask_stack = np.zeros(array_shape + (self.constants.num_modstates,))
|
|
346
382
|
|
|
347
383
|
with self.telemetry_span(f"Correcting {self.constants.num_modstates} modstates"):
|
|
348
384
|
for modstate in range(1, self.constants.num_modstates + 1):
|
|
349
385
|
# Correct the arrays
|
|
350
|
-
corrected_array, corrected_header = self.correct_single_frame(
|
|
386
|
+
corrected_array, corrected_header, nan_mask = self.correct_single_frame(
|
|
351
387
|
beam=beam,
|
|
352
388
|
modstate=modstate,
|
|
353
389
|
raster_step=raster_step,
|
|
@@ -358,6 +394,7 @@ class ScienceCalibration(
|
|
|
358
394
|
# Add this result to the 3D stack
|
|
359
395
|
array_stack[:, :, modstate - 1] = corrected_array
|
|
360
396
|
header_stack.append(corrected_header)
|
|
397
|
+
nan_mask_stack[:, :, modstate - 1] = nan_mask
|
|
361
398
|
|
|
362
399
|
with self.telemetry_span("Applying instrument polarization correction"):
|
|
363
400
|
intermediate_array = nd_left_matrix_multiply(
|
|
@@ -366,7 +403,8 @@ class ScienceCalibration(
|
|
|
366
403
|
)
|
|
367
404
|
intermediate_header = self.compute_date_keys(header_stack)
|
|
368
405
|
|
|
369
|
-
|
|
406
|
+
# The modulator state NaN masks are stacked along axis=2 with axis=0 & 1 being the array axes of one modstate
|
|
407
|
+
return intermediate_array, intermediate_header, np.logical_or.reduce(nan_mask_stack, axis=2)
|
|
370
408
|
|
|
371
409
|
def combine_beams(
|
|
372
410
|
self,
|
|
@@ -440,6 +478,7 @@ class ScienceCalibration(
|
|
|
440
478
|
calibrated_object: VispL0FitsAccess,
|
|
441
479
|
map_scan: int,
|
|
442
480
|
calibrations: CalibrationCollection,
|
|
481
|
+
nan_mask: np.ndarray,
|
|
443
482
|
) -> None:
|
|
444
483
|
"""
|
|
445
484
|
Write out calibrated science frames.
|
|
@@ -458,6 +497,9 @@ class ScienceCalibration(
|
|
|
458
497
|
calibrations
|
|
459
498
|
Calibration collection
|
|
460
499
|
|
|
500
|
+
nan_mask
|
|
501
|
+
A mask containing the known areas where data does not exist for both beams
|
|
502
|
+
|
|
461
503
|
Returns
|
|
462
504
|
-------
|
|
463
505
|
None
|
|
@@ -473,7 +515,8 @@ class ScienceCalibration(
|
|
|
473
515
|
stokes_I_data = calibrated_object.data[:, :, 0]
|
|
474
516
|
for i, stokes_param in enumerate(self.constants.stokes_params):
|
|
475
517
|
stokes_data = calibrated_object.data[:, :, i]
|
|
476
|
-
|
|
518
|
+
nan_masked_data = np.where(nan_mask, np.nan, stokes_data)
|
|
519
|
+
final_data = self.re_dummy_data(nan_masked_data)
|
|
477
520
|
pol_header = self.add_L1_pol_headers(final_header, stokes_data, stokes_I_data)
|
|
478
521
|
self.write_cal_array(
|
|
479
522
|
data=final_data,
|
|
@@ -483,7 +526,8 @@ class ScienceCalibration(
|
|
|
483
526
|
map_scan=map_scan,
|
|
484
527
|
)
|
|
485
528
|
else: # Only write stokes I
|
|
486
|
-
|
|
529
|
+
nan_masked_data = np.where(nan_mask, np.nan, calibrated_object.data)
|
|
530
|
+
final_data = self.re_dummy_data(nan_masked_data)
|
|
487
531
|
self.write_cal_array(
|
|
488
532
|
data=final_data,
|
|
489
533
|
header=final_header,
|
|
@@ -500,7 +544,7 @@ class ScienceCalibration(
|
|
|
500
544
|
map_scan: int,
|
|
501
545
|
readout_exp_time: float,
|
|
502
546
|
calibrations: CalibrationCollection,
|
|
503
|
-
) -> tuple[np.ndarray, fits.Header]:
|
|
547
|
+
) -> tuple[np.ndarray, fits.Header, np.ndarray]:
|
|
504
548
|
"""
|
|
505
549
|
Apply basic corrections to a single frame.
|
|
506
550
|
|
|
@@ -595,7 +639,40 @@ class ScienceCalibration(
|
|
|
595
639
|
self.corrections_remove_spec_geometry(geo_corrected_array, spec_shift)
|
|
596
640
|
)
|
|
597
641
|
|
|
598
|
-
|
|
642
|
+
nan_mask = self.generate_nan_mask(
|
|
643
|
+
solar_corrected_array=solar_corrected_array,
|
|
644
|
+
state_offset=state_offset,
|
|
645
|
+
angle=angle,
|
|
646
|
+
spec_shift=spec_shift,
|
|
647
|
+
)
|
|
648
|
+
|
|
649
|
+
return (
|
|
650
|
+
spectral_corrected_array,
|
|
651
|
+
observe_object.header,
|
|
652
|
+
nan_mask,
|
|
653
|
+
)
|
|
654
|
+
|
|
655
|
+
def generate_nan_mask(
|
|
656
|
+
self,
|
|
657
|
+
solar_corrected_array: np.ndarray,
|
|
658
|
+
state_offset: np.ndarray,
|
|
659
|
+
angle: float,
|
|
660
|
+
spec_shift: np.ndarray,
|
|
661
|
+
) -> np.ndarray:
|
|
662
|
+
"""Calculate the NaN mask through geometric correction to be applied to the final L1 arrays."""
|
|
663
|
+
# Using a bi-cubic polynomial (order = 3) best converges to the desired result in the underlying fits
|
|
664
|
+
geo_corrected_with_nan = next(
|
|
665
|
+
self.corrections_correct_geometry(
|
|
666
|
+
solar_corrected_array, state_offset, angle, mode="constant", order=3, cval=np.nan
|
|
667
|
+
)
|
|
668
|
+
)
|
|
669
|
+
# Interpolating with nearest neighbor (order = 0) prevents NaN values from "taking over" the whole array
|
|
670
|
+
spectral_corrected_with_nan = next(
|
|
671
|
+
self.corrections_remove_spec_geometry(
|
|
672
|
+
geo_corrected_with_nan, spec_shift, cval=np.nan, order=0
|
|
673
|
+
)
|
|
674
|
+
)
|
|
675
|
+
return np.isnan(spectral_corrected_with_nan)
|
|
599
676
|
|
|
600
677
|
def telescope_polarization_correction(
|
|
601
678
|
self,
|
|
@@ -364,10 +364,10 @@ def test_science_calibration_task(
|
|
|
364
364
|
assert header["VSPMAP"] == map_scan
|
|
365
365
|
|
|
366
366
|
# Check that WCS keys were updated
|
|
367
|
-
if offsets[1, 0, 0]
|
|
368
|
-
assert header["CRPIX2"] == input_header["CRPIX2"] - np.ceil(offsets[1, 0, 0])
|
|
369
|
-
if offsets[1, 0, 1]
|
|
370
|
-
assert header["CRPIX1"] == input_header["CRPIX1"] - np.ceil(offsets[1, 0, 1])
|
|
367
|
+
if offsets[1, 0, 0] < 0:
|
|
368
|
+
assert header["CRPIX2"] == input_header["CRPIX2"] - np.ceil(-offsets[1, 0, 0])
|
|
369
|
+
if offsets[1, 0, 1] < 0:
|
|
370
|
+
assert header["CRPIX1"] == input_header["CRPIX1"] - np.ceil(-offsets[1, 0, 1])
|
|
371
371
|
|
|
372
372
|
quality_files = task.read(tags=[Tag.quality("TASK_TYPES")])
|
|
373
373
|
for file in quality_files:
|
|
@@ -420,7 +420,7 @@ def test_readout_normalization_correct(
|
|
|
420
420
|
)
|
|
421
421
|
|
|
422
422
|
# When:
|
|
423
|
-
corrected_array, _ = task.correct_single_frame(
|
|
423
|
+
corrected_array, _, _ = task.correct_single_frame(
|
|
424
424
|
beam=1,
|
|
425
425
|
modstate=1,
|
|
426
426
|
raster_step=1,
|
|
@@ -513,7 +513,7 @@ def test_compute_date_keys_compressed_headers(
|
|
|
513
513
|
[[1.0, 2.0], [11.0, 10.0], [3.0, 2.0]], # Beam 2
|
|
514
514
|
]
|
|
515
515
|
),
|
|
516
|
-
[slice(
|
|
516
|
+
[slice(0, -11, None), slice(0, -10, None)],
|
|
517
517
|
),
|
|
518
518
|
(
|
|
519
519
|
np.array(
|
|
@@ -522,7 +522,7 @@ def test_compute_date_keys_compressed_headers(
|
|
|
522
522
|
[[-1.0, -2.0], [-11.0, -10.0], [-3.0, -2.0]], # Beam 2
|
|
523
523
|
]
|
|
524
524
|
),
|
|
525
|
-
[slice(
|
|
525
|
+
[slice(11, None, None), slice(10, None, None)],
|
|
526
526
|
),
|
|
527
527
|
(
|
|
528
528
|
np.array(
|
|
@@ -531,7 +531,7 @@ def test_compute_date_keys_compressed_headers(
|
|
|
531
531
|
[[1.0, 2.0], [-11.0, 10.0], [-3.0, -2.0]], # Beam 2
|
|
532
532
|
]
|
|
533
533
|
),
|
|
534
|
-
[slice(
|
|
534
|
+
[slice(11, -10, None), slice(2, -10, None)],
|
|
535
535
|
),
|
|
536
536
|
],
|
|
537
537
|
ids=["All positive", "All negative", "Positive and negative"],
|
|
@@ -581,3 +581,98 @@ def test_combine_beams(
|
|
|
581
581
|
expected = np.ones((10, 10, 4)) * 2.5
|
|
582
582
|
|
|
583
583
|
np.testing.assert_array_equal(data, expected)
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
@pytest.mark.parametrize(
|
|
587
|
+
"shifts",
|
|
588
|
+
# Shifts have shape (num_beams, num_modstates, 2)
|
|
589
|
+
# So the inner-most lists below (e.g., [5.0, 6.0]) correspond to [x_shift, y_shit]
|
|
590
|
+
[
|
|
591
|
+
np.array(
|
|
592
|
+
[
|
|
593
|
+
[[0.0, 0.0], [10.0, 2.0], [5.0, 6.0]], # Beam 1
|
|
594
|
+
[[1.0, 2.0], [-11.0, 10.0], [-3.0, -2.0]], # Beam 2
|
|
595
|
+
]
|
|
596
|
+
),
|
|
597
|
+
],
|
|
598
|
+
ids=["Positive and negative"],
|
|
599
|
+
)
|
|
600
|
+
def test_combine_and_cut_nan_masks(
|
|
601
|
+
science_calibration_task, calibration_collection_with_geo_shifts, shifts
|
|
602
|
+
):
|
|
603
|
+
"""
|
|
604
|
+
Given: A ScienceCalibration task and NaN masks, along with geometric shifts
|
|
605
|
+
When: Combining the two NaN masks
|
|
606
|
+
Then: The final mask has NaN values in the correct place and is correctly cropped
|
|
607
|
+
"""
|
|
608
|
+
nan_1_location = [0, 1]
|
|
609
|
+
nan_2_location = [50, 50]
|
|
610
|
+
nan_3_location = [4, 1]
|
|
611
|
+
nan_4_location = [55, 63]
|
|
612
|
+
nan_mask_shape = (100, 100)
|
|
613
|
+
nan_mask_1 = np.zeros(shape=nan_mask_shape)
|
|
614
|
+
nan_mask_1[nan_1_location[0], nan_1_location[1]] = np.nan
|
|
615
|
+
nan_mask_1[nan_2_location[0], nan_2_location[1]] = np.nan
|
|
616
|
+
nan_mask_2 = np.zeros(shape=nan_mask_shape)
|
|
617
|
+
nan_mask_2[nan_3_location[0], nan_3_location[1]] = np.nan
|
|
618
|
+
nan_mask_2[nan_4_location[0], nan_4_location[1]] = np.nan
|
|
619
|
+
task, _, _, _, _, _ = science_calibration_task
|
|
620
|
+
combined_nan_mask = task.combine_and_cut_nan_masks(
|
|
621
|
+
nan_masks=[nan_mask_1, nan_mask_2], calibrations=calibration_collection_with_geo_shifts
|
|
622
|
+
)
|
|
623
|
+
beam_1_shifts = shifts[0]
|
|
624
|
+
beam_2_shifts = shifts[1]
|
|
625
|
+
beam_1_x_shifts = [i[0] for i in beam_1_shifts]
|
|
626
|
+
beam_2_x_shifts = [i[0] for i in beam_2_shifts]
|
|
627
|
+
beam_1_y_shifts = [i[1] for i in beam_1_shifts]
|
|
628
|
+
beam_2_y_shifts = [i[1] for i in beam_2_shifts]
|
|
629
|
+
x_shifts = beam_1_x_shifts + beam_2_x_shifts
|
|
630
|
+
y_shifts = beam_1_y_shifts + beam_2_y_shifts
|
|
631
|
+
assert combined_nan_mask.shape == (
|
|
632
|
+
nan_mask_shape[0] - (max(x_shifts) - min(x_shifts)),
|
|
633
|
+
nan_mask_shape[1] - (max(y_shifts) - min(y_shifts)),
|
|
634
|
+
)
|
|
635
|
+
# Check that one NaN value from each original mask is present in the combined mask and in the correct place
|
|
636
|
+
assert (
|
|
637
|
+
combined_nan_mask[
|
|
638
|
+
nan_2_location[0] - int(abs(min(x_shifts))), nan_2_location[1] - int(abs(min(y_shifts)))
|
|
639
|
+
]
|
|
640
|
+
== True
|
|
641
|
+
)
|
|
642
|
+
assert (
|
|
643
|
+
combined_nan_mask[
|
|
644
|
+
nan_4_location[0] - int(abs(min(x_shifts))), nan_4_location[1] - int(abs(min(y_shifts)))
|
|
645
|
+
]
|
|
646
|
+
== True
|
|
647
|
+
)
|
|
648
|
+
assert np.sum(combined_nan_mask) == 2 # only two NaN values are in the final mask
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def test_generate_nan_mask(science_calibration_task, dummy_calibration_collection):
|
|
652
|
+
"""
|
|
653
|
+
Given: a calibration collection
|
|
654
|
+
When: calculating the NaN mask to use
|
|
655
|
+
Then: the mask takes up some, but not all, of the frame size
|
|
656
|
+
"""
|
|
657
|
+
task, _, _, _, _, _ = science_calibration_task
|
|
658
|
+
calibration_collection, _, _ = dummy_calibration_collection
|
|
659
|
+
beam = 1
|
|
660
|
+
modstate = 1
|
|
661
|
+
solar_gain_array = calibration_collection.solar_gain[VispTag.beam(beam)][
|
|
662
|
+
VispTag.modstate(modstate)
|
|
663
|
+
]
|
|
664
|
+
angle = calibration_collection.angle[VispTag.beam(beam)]
|
|
665
|
+
spec_shift = calibration_collection.spec_shift[VispTag.beam(beam)]
|
|
666
|
+
state_offset = calibration_collection.state_offset[VispTag.beam(beam)][
|
|
667
|
+
VispTag.modstate(modstate)
|
|
668
|
+
]
|
|
669
|
+
nan_mask = task.generate_nan_mask(
|
|
670
|
+
solar_corrected_array=np.random.random(size=solar_gain_array.shape),
|
|
671
|
+
state_offset=state_offset,
|
|
672
|
+
angle=angle,
|
|
673
|
+
spec_shift=spec_shift,
|
|
674
|
+
)
|
|
675
|
+
# Some of the mask is marked as NaN but not all
|
|
676
|
+
assert np.sum(nan_mask) < np.size(nan_mask)
|
|
677
|
+
# Ensure that only zeroes and ones are in the mask
|
|
678
|
+
assert set(np.unique(nan_mask)) == {0, 1}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dkist-processing-visp
|
|
3
|
-
Version:
|
|
3
|
+
Version: 4.0.0
|
|
4
4
|
Summary: Science processing code for the ViSP instrument on DKIST
|
|
5
5
|
Author-email: NSO / AURA <dkistdc@nso.edu>
|
|
6
6
|
License: BSD-3-Clause
|
|
@@ -116,19 +116,19 @@ Requires-Dist: asdf==3.5.0; extra == "frozen"
|
|
|
116
116
|
Requires-Dist: asdf_standard==1.4.0; extra == "frozen"
|
|
117
117
|
Requires-Dist: asdf_transform_schemas==0.6.0; extra == "frozen"
|
|
118
118
|
Requires-Dist: asgiref==3.10.0; extra == "frozen"
|
|
119
|
-
Requires-Dist: asteval==1.0.
|
|
119
|
+
Requires-Dist: asteval==1.0.7; extra == "frozen"
|
|
120
120
|
Requires-Dist: astropy==7.0.2; extra == "frozen"
|
|
121
|
-
Requires-Dist: astropy-iers-data==0.2025.11.
|
|
121
|
+
Requires-Dist: astropy-iers-data==0.2025.11.10.0.38.31; extra == "frozen"
|
|
122
122
|
Requires-Dist: asyncpg==0.30.0; extra == "frozen"
|
|
123
123
|
Requires-Dist: attrs==25.4.0; extra == "frozen"
|
|
124
124
|
Requires-Dist: babel==2.17.0; extra == "frozen"
|
|
125
125
|
Requires-Dist: billiard==4.2.2; extra == "frozen"
|
|
126
126
|
Requires-Dist: blinker==1.9.0; extra == "frozen"
|
|
127
|
-
Requires-Dist: boto3==1.40.
|
|
128
|
-
Requires-Dist: botocore==1.40.
|
|
127
|
+
Requires-Dist: boto3==1.40.71; extra == "frozen"
|
|
128
|
+
Requires-Dist: botocore==1.40.71; extra == "frozen"
|
|
129
129
|
Requires-Dist: cachelib==0.13.0; extra == "frozen"
|
|
130
130
|
Requires-Dist: celery==5.3.1; extra == "frozen"
|
|
131
|
-
Requires-Dist: certifi==2025.
|
|
131
|
+
Requires-Dist: certifi==2025.11.12; extra == "frozen"
|
|
132
132
|
Requires-Dist: cffi==2.0.0; extra == "frozen"
|
|
133
133
|
Requires-Dist: charset-normalizer==3.4.4; extra == "frozen"
|
|
134
134
|
Requires-Dist: click==8.3.0; extra == "frozen"
|
|
@@ -152,7 +152,7 @@ Requires-Dist: dkist-processing-common==11.8.0; extra == "frozen"
|
|
|
152
152
|
Requires-Dist: dkist-processing-core==6.0.0; extra == "frozen"
|
|
153
153
|
Requires-Dist: dkist-processing-math==2.2.1; extra == "frozen"
|
|
154
154
|
Requires-Dist: dkist-processing-pac==3.1.1; extra == "frozen"
|
|
155
|
-
Requires-Dist: dkist-processing-visp==
|
|
155
|
+
Requires-Dist: dkist-processing-visp==4.0.0; extra == "frozen"
|
|
156
156
|
Requires-Dist: dkist-service-configuration==4.1.7; extra == "frozen"
|
|
157
157
|
Requires-Dist: dkist-spectral-lines==3.0.0; extra == "frozen"
|
|
158
158
|
Requires-Dist: dkist_fits_specifications==4.17.0; extra == "frozen"
|
|
@@ -164,10 +164,9 @@ Requires-Dist: fonttools==4.60.1; extra == "frozen"
|
|
|
164
164
|
Requires-Dist: frozenlist==1.8.0; extra == "frozen"
|
|
165
165
|
Requires-Dist: fsspec==2025.10.0; extra == "frozen"
|
|
166
166
|
Requires-Dist: globus-sdk==3.65.0; extra == "frozen"
|
|
167
|
-
Requires-Dist: google-re2==1.1.
|
|
168
|
-
Requires-Dist: googleapis-common-protos==1.
|
|
167
|
+
Requires-Dist: google-re2==1.1.20251105; extra == "frozen"
|
|
168
|
+
Requires-Dist: googleapis-common-protos==1.72.0; extra == "frozen"
|
|
169
169
|
Requires-Dist: gqlclient==1.2.3; extra == "frozen"
|
|
170
|
-
Requires-Dist: greenlet==3.2.4; extra == "frozen"
|
|
171
170
|
Requires-Dist: grpcio==1.76.0; extra == "frozen"
|
|
172
171
|
Requires-Dist: gunicorn==23.0.0; extra == "frozen"
|
|
173
172
|
Requires-Dist: h11==0.16.0; extra == "frozen"
|
|
@@ -254,9 +253,9 @@ Requires-Dist: protobuf==6.33.0; extra == "frozen"
|
|
|
254
253
|
Requires-Dist: psutil==7.1.3; extra == "frozen"
|
|
255
254
|
Requires-Dist: psycopg2-binary==2.9.11; extra == "frozen"
|
|
256
255
|
Requires-Dist: pycparser==2.23; extra == "frozen"
|
|
257
|
-
Requires-Dist: pydantic==2.12.
|
|
258
|
-
Requires-Dist: pydantic-settings==2.
|
|
259
|
-
Requires-Dist: pydantic_core==2.41.
|
|
256
|
+
Requires-Dist: pydantic==2.12.4; extra == "frozen"
|
|
257
|
+
Requires-Dist: pydantic-settings==2.12.0; extra == "frozen"
|
|
258
|
+
Requires-Dist: pydantic_core==2.41.5; extra == "frozen"
|
|
260
259
|
Requires-Dist: pyerfa==2.0.1.5; extra == "frozen"
|
|
261
260
|
Requires-Dist: pyparsing==3.2.5; extra == "frozen"
|
|
262
261
|
Requires-Dist: python-daemon==3.1.2; extra == "frozen"
|
|
@@ -300,7 +299,7 @@ Requires-Dist: typing_extensions==4.15.0; extra == "frozen"
|
|
|
300
299
|
Requires-Dist: tzdata==2025.2; extra == "frozen"
|
|
301
300
|
Requires-Dist: uc-micro-py==1.0.3; extra == "frozen"
|
|
302
301
|
Requires-Dist: uncertainties==3.2.3; extra == "frozen"
|
|
303
|
-
Requires-Dist: universal_pathlib==0.3.
|
|
302
|
+
Requires-Dist: universal_pathlib==0.3.5; extra == "frozen"
|
|
304
303
|
Requires-Dist: urllib3==2.5.0; extra == "frozen"
|
|
305
304
|
Requires-Dist: vine==5.1.0; extra == "frozen"
|
|
306
305
|
Requires-Dist: voluptuous==0.15.2; extra == "frozen"
|
|
@@ -24,16 +24,16 @@ dkist_processing_visp/tasks/geometric.py,sha256=rvWa13T2_cPQtiMjlWrqMYS5UvL8Xgmi
|
|
|
24
24
|
dkist_processing_visp/tasks/instrument_polarization.py,sha256=uj7iyzM3CiJcbQeF4eKpk_KCoheXaM4FpDI83GYDld4,25854
|
|
25
25
|
dkist_processing_visp/tasks/l1_output_data.py,sha256=lon6bIUBvURV_7gn1HFGAGVuEiLoOiMaJwfbdhSRW3w,459
|
|
26
26
|
dkist_processing_visp/tasks/lamp.py,sha256=RciNB8zW5fDx5yrsaNZgFP7LsajmSvkTNvbe3Sm6tjc,6060
|
|
27
|
-
dkist_processing_visp/tasks/make_movie_frames.py,sha256=
|
|
27
|
+
dkist_processing_visp/tasks/make_movie_frames.py,sha256=fw25ksKiJJNS57XV5a7rHpYGcSkYxS2Qf13Fb1UGNpE,7544
|
|
28
28
|
dkist_processing_visp/tasks/parse.py,sha256=3f8LWSKQtuY7n-PdHiw1j55i5l4juBn3pIHqTtp2-rk,6752
|
|
29
|
-
dkist_processing_visp/tasks/quality_metrics.py,sha256=
|
|
30
|
-
dkist_processing_visp/tasks/science.py,sha256=
|
|
29
|
+
dkist_processing_visp/tasks/quality_metrics.py,sha256=Pw55-PXW0cl39FuNkEQCGGhvI_zMDimwmh-swVPVBD4,8133
|
|
30
|
+
dkist_processing_visp/tasks/science.py,sha256=BiEtgDowExHAXXzfO4BSr34BT0PZ28CdtOJMP4TDA20,34680
|
|
31
31
|
dkist_processing_visp/tasks/solar.py,sha256=QPkrJqUz6gYxFFvNlATMKa4NIEdn4iassYn86qPb60k,27567
|
|
32
32
|
dkist_processing_visp/tasks/visp_base.py,sha256=flUM-dCWkjcGvfaiFWH89DRZXz7NADgShxzpFjFqVEw,1370
|
|
33
33
|
dkist_processing_visp/tasks/write_l1.py,sha256=bsDZ0BwoqpTtS_f_rAzUn7Ra8UvYb-kINQhX6BwwFQw,8796
|
|
34
34
|
dkist_processing_visp/tasks/mixin/__init__.py,sha256=z2nFVvvIzirxklQ9i5-F1nR-WOgcDttYtog_jx4yN5I,12
|
|
35
35
|
dkist_processing_visp/tasks/mixin/beam_access.py,sha256=1VSJkH6yMxCiZWdWOp_RJ37fX5ULMYmB_0_ulT7YJpI,870
|
|
36
|
-
dkist_processing_visp/tasks/mixin/corrections.py,sha256=
|
|
36
|
+
dkist_processing_visp/tasks/mixin/corrections.py,sha256=FhLFgD9ZYLZd3SaC3PFF-szrcs-zmdrUYNDUEK-h7JA,7145
|
|
37
37
|
dkist_processing_visp/tasks/mixin/downsample.py,sha256=SvKzY6HJRn-FeyG7O6HPvyOS5dmMu6uPoWkfnpPXpVw,1344
|
|
38
38
|
dkist_processing_visp/tasks/mixin/line_zones.py,sha256=5jfea9V5RJAi-834z_Y9v4fhlRFJdK1McAqO9X92bZo,4065
|
|
39
39
|
dkist_processing_visp/tests/README.rst,sha256=rnedwwg25c0lB9Me7cT7QNZA17FYlqCu9ZnjQxR5hi0,12502
|
|
@@ -54,7 +54,7 @@ dkist_processing_visp/tests/test_map_repeats.py,sha256=9g3NnvSfn1OqxxYYxTFoOIi1U
|
|
|
54
54
|
dkist_processing_visp/tests/test_parameters.py,sha256=h9EemGJf0b4ma0jLGd321untkhLkhwgo88mIRnSmxXs,4369
|
|
55
55
|
dkist_processing_visp/tests/test_parse.py,sha256=aFBbLBXdz2KJZy62gC1KC049FFfRKYqYVo-Y32mx_E4,20853
|
|
56
56
|
dkist_processing_visp/tests/test_quality.py,sha256=YW24VjEHoILseFIXZBp4-o7egT26mfT1lafzajVjXu8,6905
|
|
57
|
-
dkist_processing_visp/tests/test_science.py,sha256=
|
|
57
|
+
dkist_processing_visp/tests/test_science.py,sha256=rIoHu1hifNYj1niQaDkmyID5BQVzTEgZ5BhsdZOSaWk,23985
|
|
58
58
|
dkist_processing_visp/tests/test_solar.py,sha256=Bpkm59HI0GqrwM8LqIK7GR0pUUW0anboJkePQnofgmA,9471
|
|
59
59
|
dkist_processing_visp/tests/test_trial_create_quality_report.py,sha256=t3de9LMJOqrRaFXAvKV_5sotAfzDR8fZVIrFNRB2I_A,2779
|
|
60
60
|
dkist_processing_visp/tests/test_visp_constants.py,sha256=v8meAXdR6jJntaFtWR-N65Z-3ovr9Vf9_-BwAZv2RBc,1974
|
|
@@ -87,7 +87,7 @@ docs/requirements_table.rst,sha256=_HIbwFpDooM5n0JjiDAbFozGfJuX13smtcoujLFN4Gk,2
|
|
|
87
87
|
docs/science_calibration.rst,sha256=VN_g7xSjN-nbXhlBaFnPCbNcsc_Qu0207jEUfRAjnBE,2939
|
|
88
88
|
docs/scientific_changelog.rst,sha256=01AWBSHg8zElnodCgAq-hMxhk9CkX5rtEENx4iz0sjI,300
|
|
89
89
|
licenses/LICENSE.rst,sha256=piZaQplkzOMmH1NXg6QIdo9wwo9pPCoHkvm2-DmH76E,1462
|
|
90
|
-
dkist_processing_visp-
|
|
91
|
-
dkist_processing_visp-
|
|
92
|
-
dkist_processing_visp-
|
|
93
|
-
dkist_processing_visp-
|
|
90
|
+
dkist_processing_visp-4.0.0.dist-info/METADATA,sha256=4S8qtlI-Cp12PpvTDgD0ZgSA2ZaaOp0QnHtC94RZ4Ak,29073
|
|
91
|
+
dkist_processing_visp-4.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
92
|
+
dkist_processing_visp-4.0.0.dist-info/top_level.txt,sha256=9GHSn-ZMGQxaRNGrPP3HNc5ZkE7ftzluO74Jz5vUSTE,46
|
|
93
|
+
dkist_processing_visp-4.0.0.dist-info/RECORD,,
|
|
File without changes
|
{dkist_processing_visp-3.6.3.dist-info → dkist_processing_visp-4.0.0.dist-info}/top_level.txt
RENAMED
|
File without changes
|