dkist-processing-visp 3.3.0__py3-none-any.whl → 5.1.1__py3-none-any.whl

dkist_processing_visp/tasks/lamp.py

@@ -1,5 +1,5 @@
 """ViSP lamp calibration task. See :doc:`this page </gain_correction>` for more information."""
-import numpy as np
+
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.fits import fits_array_decoder
 from dkist_processing_common.codecs.fits import fits_array_encoder
@@ -29,13 +29,14 @@ class LampCalibration(
 
     Parameters
     ----------
-    recipe_run_id : int
+    recipe_run_id
         id of the recipe run used to identify the workflow run this task is part of
-    workflow_name : str
+
+    workflow_name
         name of the workflow to which this instance of the task belongs
-    workflow_version : str
-        version of the workflow to which this instance of the task belongs
 
+    workflow_version
+        version of the workflow to which this instance of the task belongs
     """
 
     record_provenance = True
@@ -44,9 +45,11 @@ class LampCalibration(
         """
         For each beam.
 
-            - Gather input lamp gain and averaged dark arrays
-            - Calculate master lamp
-            - Write master lamp
+            - Normalize all input arrays by the number of frames per FPA
+            - Subtract the average dark frame corresponding to the matching readout exposure time
+            - Average all different readout exposure time arrays (if applicable)
+            - Interpolate over the hairlines
+            - Write final lamp gain to disk
             - Record quality metrics
 
         Returns
@@ -54,12 +57,14 @@ class LampCalibration(
         None
 
         """
-        with self.apm_task_step(
+        with self.telemetry_span(
             f"Generate lamp gains for {self.constants.num_beams} beams and {len(self.constants.lamp_readout_exp_times)} exposure times"
         ):
-            for readout_exp_time in self.constants.lamp_readout_exp_times:
-                for beam in range(1, self.constants.num_beams + 1):
-                    logger.info(f"Load dark for beam {beam}")
+            for beam in range(1, self.constants.num_beams + 1):
+                all_exp_time_arrays = []
+                for readout_exp_time in self.constants.lamp_readout_exp_times:
+                    apm_str = f"{beam = } and {readout_exp_time = }"
+                    logger.info(f"Load dark for beam {apm_str}")
                     dark_array = next(
                         self.read(
                             tags=VispTag.intermediate_frame_dark(
@@ -69,20 +74,45 @@ class LampCalibration(
                         )
                     )
 
-                    for state_num in range(
-                        1, self.constants.num_modstates + 1
-                    ):  # modulator states go from 1 to n
-                        logger.info(
-                            f"Calculating average lamp gain for beam {beam}, modulator state {state_num}"
+                    with self.telemetry_span(f"Computing gain for {apm_str}"):
+                        tags = [
+                            VispTag.input(),
+                            VispTag.frame(),
+                            VispTag.task_lamp_gain(),
+                            VispTag.readout_exp_time(readout_exp_time),
+                        ]
+                        input_lamp_gain_objs = self.read(
+                            tags=tags,
+                            decoder=fits_access_decoder,
+                            fits_access_class=VispL0FitsAccess,
                         )
-                        self.compute_and_write_master_lamp_gain_for_modstate(
-                            modstate=state_num,
-                            dark_array=dark_array,
-                            beam=beam,
-                            readout_exp_time=readout_exp_time,
+
+                        readout_normalized_arrays = (
+                            self.beam_access_get_beam(o.data, beam=beam) / o.num_raw_frames_per_fpa
+                            for o in input_lamp_gain_objs
+                        )
+                        averaged_gain_data = average_numpy_arrays(readout_normalized_arrays)
+
+                        dark_corrected_gain_data = next(
+                            subtract_array_from_arrays(averaged_gain_data, dark_array)
                         )
 
-        with self.apm_processing_step("Computing and logging quality metrics"):
+                        all_exp_time_arrays.append(dark_corrected_gain_data)
+
+                avg_gain_array = average_numpy_arrays(all_exp_time_arrays)
+                filtered_gain_data = self.corrections_mask_hairlines(avg_gain_array)
+
+                with self.telemetry_span(f"Writing gain array for {apm_str}"):
+                    self.write(
+                        data=filtered_gain_data,
+                        tags=[
+                            VispTag.intermediate_frame(beam=beam),
+                            VispTag.task_lamp_gain(),
+                        ],
+                        encoder=fits_array_encoder,
+                    )
+
+        with self.telemetry_span("Computing and logging quality metrics"):
             no_of_raw_lamp_frames: int = self.scratch.count_all(
                 tags=[
                     VispTag.input(),
@@ -94,73 +124,3 @@ class LampCalibration(
             self.quality_store_task_type_counts(
                 task_type=TaskName.lamp_gain.value, total_frames=no_of_raw_lamp_frames
             )
-
-    def compute_and_write_master_lamp_gain_for_modstate(
-        self,
-        modstate: int,
-        dark_array: np.ndarray,
-        beam: int,
-        readout_exp_time: float,
-    ) -> None:
-        """
-        Compute and write master lamp gain for a given modstate and beam.
-
-        Generally the algorithm is:
-            1. Average input gain arrays
-            2. Subtract average dark to get the dark corrected gain data
-            3. Normalize each beam to unity mean
-            4. Write to disk
-
-        Parameters
-        ----------
-        modstate : int
-            The modulator state to calculate the master lamp gain for
-
-        dark_array : np.ndarray
-            The master dark to be subtracted from each lamp gain file
-
-        beam : int
-            The number of the beam
-
-        readout_exp_time : float
-            Exposure time of single readout
-
-        Returns
-        -------
-        None
-        """
-        apm_str = f"{beam = }, {modstate = }, and {readout_exp_time = }"
-        # Get the input lamp gain arrays
-        tags = [
-            VispTag.input(),
-            VispTag.frame(),
-            VispTag.task_lamp_gain(),
-            VispTag.modstate(modstate),
-            VispTag.readout_exp_time(readout_exp_time),
-        ]
-        input_lamp_gain_objs = self.read(
-            tags=tags, decoder=fits_access_decoder, fits_access_class=VispL0FitsAccess
-        )
-
-        with self.apm_processing_step(f"Computing gain for {apm_str}"):
-
-            readout_normalized_arrays = (
-                self.beam_access_get_beam(o.data, beam=beam) / o.num_raw_frames_per_fpa
-                for o in input_lamp_gain_objs
-            )
-            averaged_gain_data = average_numpy_arrays(readout_normalized_arrays)
-
-            dark_corrected_gain_data = next(
-                subtract_array_from_arrays(averaged_gain_data, dark_array)
-            )
-            filtered_gain_data = self.corrections_mask_hairlines(dark_corrected_gain_data)
-
-        with self.apm_writing_step(f"Writing gain array for {apm_str}"):
-            self.write(
-                data=filtered_gain_data,
-                tags=[
-                    VispTag.intermediate_frame(beam=beam, modstate=modstate),
-                    VispTag.task_lamp_gain(),
-                ],
-                encoder=fits_array_encoder,
-            )

dkist_processing_visp/tasks/make_movie_frames.py

@@ -1,9 +1,11 @@
 """Visp make movie frames task."""
+
 import numpy as np
 from astropy.io import fits
 from astropy.visualization import ZScaleInterval
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.fits import fits_array_encoder
+from dkist_processing_common.models.fits_access import MetadataKey
 from dkist_service_configuration.logging import logger
 
 from dkist_processing_visp.models.tags import VispTag
@@ -46,7 +48,7 @@ class MakeVispMovieFrames(VispTaskBase):
         stokes_states = ["I", "Q", "U", "V"]
         # Loop over the number of raster scans
         for map_scan in range(1, self.constants.num_map_scans + 1):
-            with self.apm_processing_step(f"Making movie frame for {map_scan = }"):
+            with self.telemetry_span(f"Making movie frame for {map_scan = }"):
                 instrument_set = set()
                 wavelength_set = set()
                 time_obs = []
@@ -77,7 +79,7 @@ class MakeVispMovieFrames(VispTaskBase):
                                     fits_access_class=VispL1FitsAccess,
                                 )
                             )
-                            data = calibrated_frame.data
+                            data = np.nan_to_num(calibrated_frame.data, nan=0)
                             if self.constants.num_raster_steps == 1:
                                 logger.info(
                                     "Only a single raster step found. Making a spectral movie."
@@ -120,16 +122,16 @@ class MakeVispMovieFrames(VispTaskBase):
                         f"There should only be one instrument value in the headers. "
                         f"Found {len(instrument_set)}: {instrument_set=}"
                     )
-                header["INSTRUME"] = instrument_set.pop()
+                header[MetadataKey.instrument] = instrument_set.pop()
                 # The timestamp of a movie frame will be the time of raster scan start
-                header["DATE-BEG"] = time_obs[0]
+                header[MetadataKey.time_obs] = time_obs[0]
                 # Make sure only one wavelength value was found
                 if len(wavelength_set) != 1:
                     raise ValueError(
                         f"There should only be one wavelength value in the headers. "
                         f"Found {len(wavelength_set)}: {wavelength_set=}"
                     )
-                header["LINEWAV"] = wavelength_set.pop()
+                header[MetadataKey.wavelength] = wavelength_set.pop()
                 # Write the movie frame file to disk and tag it, normalizing across stokes intensities
                 if is_polarized:
                     i_norm = ZScaleInterval()(stokes_i_data)
@@ -146,7 +148,7 @@ class MakeVispMovieFrames(VispTaskBase):
                 else:
                     movie_frame_data = stokes_i_data
 
-            with self.apm_writing_step(f"Writing movie frame for {map_scan = }"):
+            with self.telemetry_span(f"Writing movie frame for {map_scan = }"):
                 self.write(
                     data=np.asarray(movie_frame_data),
                     tags=[

dkist_processing_visp/tasks/mixin/beam_access.py

@@ -1,4 +1,5 @@
 """Helper to manage extract a given beam from input data."""
+
 import numpy as np
 
 

dkist_processing_visp/tasks/mixin/corrections.py

@@ -1,4 +1,5 @@
 """Helper for ViSP array corrections."""
+
 from typing import Generator
 from typing import Iterable
 
@@ -16,6 +17,9 @@ class CorrectionsMixin:
         arrays: Iterable[np.ndarray] | np.ndarray,
         shift: np.ndarray = np.zeros(2),
         angle: float = 0.0,
+        mode: str = "edge",
+        order: int = 5,
+        cval: float = np.nan,
     ) -> Generator[np.ndarray, None, None]:
         """
         Shift and then rotate data.
@@ -34,6 +38,24 @@ class CorrectionsMixin:
         angle : float
             The angle (in radians) between slit hairlines and pixel axes.
 
+        mode : {'constant', 'edge', 'symmetric', 'reflect', 'wrap'}
+            Points outside the boundaries of the input are filled according
+            to the given mode.  Modes match the behaviour of `numpy.pad`.
+
+        order : int
+            The order of interpolation. The order has to be in the range 0-5:
+             - 0: Nearest-neighbor
+             - 1: Bi-linear (default)
+             - 2: Bi-quadratic
+             - 3: Bi-cubic
+             - 4: Bi-quartic
+             - 5: Bi-quintic
+
+        cval : float
+            Used in conjunction with mode 'constant', the value outside
+            the image boundaries.
+
+
         Returns
         -------
         Generator
@@ -45,12 +67,21 @@ class CorrectionsMixin:
             array[np.where(array == np.inf)] = np.max(array[np.isfinite(array)])
             array[np.where(array == -np.inf)] = np.min(array[np.isfinite(array)])
             array[np.isnan(array)] = np.nanmedian(array)
-            translated = affine_transform_arrays(array, translation=-shift, mode="reflect", order=5)
-            yield next(rotate_arrays_about_point(translated, angle=-angle, mode="reflect", order=5))
+            translated = affine_transform_arrays(
+                array, translation=-shift, mode=mode, order=order, cval=cval
+            )
+            yield next(
+                rotate_arrays_about_point(
+                    translated, angle=-angle, mode=mode, order=order, cval=cval
+                )
+            )
 
     @staticmethod
     def corrections_remove_spec_geometry(
-        arrays: Iterable[np.ndarray] | np.ndarray, spec_shift: np.ndarray
+        arrays: Iterable[np.ndarray] | np.ndarray,
+        spec_shift: np.ndarray,
+        cval: float | None = None,
+        order: int = 3,
     ) -> Generator[np.ndarray, None, None]:
         """
         Remove spectral curvature.
@@ -66,6 +97,19 @@ class CorrectionsMixin:
             Array with shape (X), where X is the number of pixels in the spatial dimension.
             This dimension gives the spectral shift.
 
+        order : int
+            The order of interpolation. The order has to be in the range 0-5:
+             - 0: Nearest-neighbor
+             - 1: Bi-linear (default)
+             - 2: Bi-quadratic
+             - 3: Bi-cubic
+             - 4: Bi-quartic
+             - 5: Bi-quintic
+
+        cval : float
+            Used in conjunction with mode 'constant', the value outside
+            the image boundaries.
+
         Returns
         -------
         Generator
@@ -77,8 +121,14 @@ class CorrectionsMixin:
             numy = array.shape[1]
             array_output = np.zeros(array.shape)
             for j in range(numy):
+                if cval is None:
+                    cval = np.nanmedian(array[:, j])
                 array_output[:, j] = spnd.shift(
-                    array[:, j], -spec_shift[j], mode="constant", cval=np.nanmedian(array[:, j])
+                    array[:, j],
+                    -spec_shift[j],
+                    mode="constant",
+                    cval=cval,
+                    order=order,
                 )
             yield array_output
 

dkist_processing_visp/tasks/mixin/downsample.py

@@ -1,4 +1,5 @@
 """Helper mixin for downsampling data."""
+
 import numpy as np
 
 

dkist_processing_visp/tasks/parse.py

@@ -1,24 +1,28 @@
 """ViSP parse task."""
+
 from typing import TypeVar
 
+from dkist_processing_common.models.fits_access import MetadataKey
 from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_common.parsers.cs_step import CSStepFlower
 from dkist_processing_common.parsers.cs_step import NumCSStepBud
 from dkist_processing_common.parsers.retarder import RetarderNameBud
-from dkist_processing_common.parsers.task import parse_header_ip_task_with_gains
 from dkist_processing_common.parsers.task import PolcalTaskFlower
 from dkist_processing_common.parsers.task import TaskTypeFlower
+from dkist_processing_common.parsers.task import parse_header_ip_task_with_gains
 from dkist_processing_common.parsers.time import ExposureTimeFlower
 from dkist_processing_common.parsers.time import ObsIpStartTimeBud
 from dkist_processing_common.parsers.time import ReadoutExpTimeFlower
 from dkist_processing_common.parsers.time import TaskExposureTimesBud
 from dkist_processing_common.parsers.time import TaskReadoutExpTimesBud
+from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
 from dkist_processing_common.parsers.unique_bud import UniqueBud
 from dkist_processing_common.parsers.wavelength import ObserveWavelengthBud
 from dkist_processing_common.tasks import ParseL0InputDataBase
 
 from dkist_processing_visp.models.constants import VispBudName
+from dkist_processing_visp.models.fits_access import VispMetadataKey
 from dkist_processing_visp.models.parameters import VispParsingParameters
 from dkist_processing_visp.parsers.map_repeats import MapScanFlower
 from dkist_processing_visp.parsers.map_repeats import NumMapScansBud
@@ -27,6 +31,8 @@ from dkist_processing_visp.parsers.modulator_states import NumberModulatorStates
 from dkist_processing_visp.parsers.polarimeter_mode import PolarimeterModeBud
 from dkist_processing_visp.parsers.raster_step import RasterScanStepFlower
 from dkist_processing_visp.parsers.raster_step import TotalRasterStepsBud
+from dkist_processing_visp.parsers.spectrograph_configuration import IncidentLightAngleBud
+from dkist_processing_visp.parsers.spectrograph_configuration import ReflectedLightAngleBud
 from dkist_processing_visp.parsers.time import DarkReadoutExpTimePickyBud
 from dkist_processing_visp.parsers.time import NonDarkNonPolcalTaskReadoutExpTimesBud
 from dkist_processing_visp.parsers.visp_l0_fits_access import VispL0FitsAccess
@@ -72,6 +78,7 @@ class ParseL0VispInputData(ParseL0InputDataBase):
     def constant_buds(self) -> list[S]:
         """Add ViSP specific constants to common constants."""
         return super().constant_buds + [
+            UniqueBud(constant_name=VispBudName.arm_id.value, metadata_key=VispMetadataKey.arm_id),
             NumMapScansBud(),
             TotalRasterStepsBud(),
             NumCSStepBud(self.parameters.max_cs_step_time_sec),
@@ -82,6 +89,20 @@ class ParseL0VispInputData(ParseL0InputDataBase):
             RetarderNameBud(),
             NonDarkNonPolcalTaskReadoutExpTimesBud(),
             DarkReadoutExpTimePickyBud(),
+            IncidentLightAngleBud(),
+            ReflectedLightAngleBud(),
+            TaskUniqueBud(
+                constant_name=VispBudName.grating_constant_inverse_mm.value,
+                metadata_key=VispMetadataKey.grating_constant_inverse_mm,
+                ip_task_types=[TaskName.observe.value, TaskName.solar_gain.value],
+                task_type_parsing_function=parse_header_ip_task_with_gains,
+            ),
+            TaskUniqueBud(
+                constant_name=VispBudName.solar_gain_ip_start_time.value,
+                metadata_key=MetadataKey.ip_start_time,
+                ip_task_types=TaskName.solar_gain,
+                task_type_parsing_function=parse_header_ip_task_with_gains,
+            ),
             TaskExposureTimesBud(
                 stem_name=VispBudName.lamp_exposure_times.value,
                 ip_task_types=TaskName.lamp_gain.value,
@@ -122,9 +143,18 @@ class ParseL0VispInputData(ParseL0InputDataBase):
                 ip_task_types=TaskName.polcal.value,
                 header_task_parsing_func=parse_header_ip_task_with_gains,
             ),
-            UniqueBud(constant_name=VispBudName.axis_1_type.value, metadata_key="axis_1_type"),
-            UniqueBud(constant_name=VispBudName.axis_2_type.value, metadata_key="axis_2_type"),
-            UniqueBud(constant_name=VispBudName.axis_3_type.value, metadata_key="axis_3_type"),
+            UniqueBud(
+                constant_name=VispBudName.axis_1_type.value,
+                metadata_key=VispMetadataKey.axis_1_type,
+            ),
+            UniqueBud(
+                constant_name=VispBudName.axis_2_type.value,
+                metadata_key=VispMetadataKey.axis_2_type,
+            ),
+            UniqueBud(
+                constant_name=VispBudName.axis_3_type.value,
+                metadata_key=VispMetadataKey.axis_3_type,
+            ),
         ]
 
     @property

dkist_processing_visp/tasks/quality_metrics.py

@@ -1,4 +1,5 @@
 """ViSP quality metrics task."""
+
 from dataclasses import dataclass
 from dataclasses import field
 from typing import Iterable
@@ -105,7 +106,7 @@ class VispL1QualityMetrics(VispTaskBase, QualityMixin):
 
     def compute_sensitivity(self) -> None:
         """Compute RMS noise and sensitivity estimate for L1 Visp frames."""
-        with self.apm_processing_step("Calculating polarization metrics"):
+        with self.telemetry_span("Calculating polarization metrics"):
             all_datetimes = []
             all_I_sensitivity = []
             all_Q_sensitivity = []
@@ -161,7 +162,7 @@ class VispL1QualityMetrics(VispTaskBase, QualityMixin):
                             continue
 
                         # compute sensitivity for this Stokes parameter
-                        data_list.append(np.std(stokes_frame.data) / stokesI_med)
+                        data_list.append(np.nanstd(stokes_frame.data) / stokesI_med)
 
                 all_datetimes.append(Time(np.mean(polarization_data.datetimes), format="mjd").isot)
                 for target, source in zip(
@@ -173,7 +174,7 @@ class VispL1QualityMetrics(VispTaskBase, QualityMixin):
                         continue
                     target.append(np.mean(source))
 
-        with self.apm_step("Sending lists for storage"):
+        with self.telemetry_span("Sending lists for storage"):
             for stokes_index, stokes_noise in zip(
                 ("I", "Q", "U", "V"),
                 (all_I_sensitivity, all_Q_sensitivity, all_U_sensitivity, all_V_sensitivity),
@@ -186,7 +187,7 @@ class VispL1QualityMetrics(VispTaskBase, QualityMixin):
 
     def compute_noise(self):
         """Compute noise in data."""
-        with self.apm_processing_step("Calculating L1 ViSP noise metrics"):
+        with self.telemetry_span("Calculating L1 ViSP noise metrics"):
             for stokes in ["I", "Q", "U", "V"]:
                 tags = [VispTag.calibrated(), VispTag.frame(), VispTag.stokes(stokes)]
                 if self.scratch.count_all(tags=tags) > 0: