dkist-processing-visp 4.0.0__py3-none-any.whl → 5.0.0__py3-none-any.whl

dkist_processing_visp/tasks/l1_output_data.py

@@ -1,14 +1,216 @@
 """Subclass of AssembleQualityData that causes the correct polcal metrics to build."""
 
+import numpy as np
+from dkist_processing_common.codecs.asdf import asdf_decoder
+from dkist_processing_common.models.quality import Plot2D
+from dkist_processing_common.models.quality import ReportMetric
+from dkist_processing_common.models.quality import VerticalMultiPanePlot2D
 from dkist_processing_common.tasks import AssembleQualityData
 
 __all__ = ["VispAssembleQualityData"]
 
+from dkist_processing_visp.models.constants import VispConstants
+from dkist_processing_visp.models.metric_code import VispMetricCode
+from dkist_processing_visp.models.tags import VispTag
+
 
 class VispAssembleQualityData(AssembleQualityData):
     """Subclass just so that the polcal_label_list can be populated."""
 
+    constants: VispConstants
+
+    @property
+    def constants_model_class(self):
+        """Get ViSP pipeline constants."""
+        return VispConstants
+
     @property
     def polcal_label_list(self) -> list[str]:
         """Return labels for beams 1 and 2."""
         return ["Beam 1", "Beam 2"]
+
+    def quality_assemble_data(self, polcal_label_list: list[str] | None = None) -> list[dict]:
+        """
+        Assemble the full quality report and insert ViSP-specific metrics.
+
+        We try to place the new metrics right before default polcal ones, if possible.
+        """
+        vignette_metrics = []
+        for beam in range(1, self.constants.num_beams + 1):
+            vignette_metrics.append(self.build_first_vignette_metric(beam=beam))
+            vignette_metrics.append(self.build_final_vignette_metric(beam=beam))
+
+        report = super().quality_assemble_data(polcal_label_list=polcal_label_list)
+
+        # Look for the first "PolCal" metric
+        first_polcal_metric_index = 0
+        try:
+            while not report[first_polcal_metric_index]["name"].lower().startswith("polcal"):
+                first_polcal_metric_index += 1
+        except:
+            # Wasn't found for whatever reason. No big deal, just put the new metrics at the front of the list
+            first_polcal_metric_index = 0
+
+        final_report = (
+            report[:first_polcal_metric_index]
+            + vignette_metrics
+            + report[first_polcal_metric_index:]
+        )
+
+        return final_report
+
+    def build_first_vignette_metric(self, beam: int) -> dict:
+        """Build a ReportMetric showing the initial atlas-with-continuum fit and residuals."""
+        data = next(
+            self.read(
+                tags=[VispTag.quality(VispMetricCode.solar_first_vignette), VispTag.beam(beam)],
+                decoder=asdf_decoder,
+            )
+        )
+
+        wave_vec = data["output_wave_vec"].tolist()
+        input_spectrum = data["input_spectrum"].tolist()
+        best_fit_atlas = data["best_fit_atlas"].tolist()
+        continuum = data["best_fit_continuum"].tolist()
+        residuals = data["residuals"].tolist()
+
+        fit_series = {
+            "Raw input spectrum": [wave_vec, input_spectrum],
+            "Best fit atlas": [wave_vec, best_fit_atlas],
+            "Best fit continuum": [wave_vec, continuum],
+        }
+
+        fit_plot_kwargs = {
+            "Raw input spectrum": {
+                "ls": "-",
+                "ms": 0,
+                "color": "#FAA61C",
+                "zorder": 2.0,
+                "lw": 4,
+                "alpha": 0.6,
+            },
+            "Best fit atlas": {"color": "k", "ls": "-", "ms": 0, "zorder": 2.1},
+            "Best fit continuum": {"ls": "-", "ms": 0, "color": "g", "zorder": 2.2},
+        }
+
+        fit_plot = Plot2D(
+            xlabel="Wavelength [nm]",
+            ylabel="Signal",
+            series_data=fit_series,
+            plot_kwargs=fit_plot_kwargs,
+            sort_series=False,
+        )
+
+        residuals_series = {"Residuals": [wave_vec, residuals]}
+        residuals_plot_kwargs = {"Residuals": {"ls": "-", "color": "k", "ms": 0}}
+
+        y_min = np.nanpercentile(residuals, 2)
+        y_max = np.nanpercentile(residuals, 98)
+        y_range = y_max - y_min
+        y_min -= 0.1 * y_range
+        y_max += 0.1 * y_range
+        residuals_plot = Plot2D(
+            xlabel="Wavelength [nm]",
+            ylabel=r"$\frac{\mathrm{Obs - Atlas}}{\mathrm{Obs}}$",
+            series_data=residuals_series,
+            plot_kwargs=residuals_plot_kwargs,
+            ylim=(y_min, y_max),
+        )
+
+        plot_list = [fit_plot, residuals_plot]
+        height_ratios = [1.5, 1.0]
+
+        full_plot = VerticalMultiPanePlot2D(
+            top_to_bottom_plot_list=plot_list,
+            match_x_axes=True,
+            no_gap=True,
+            top_to_bottom_height_ratios=height_ratios,
+        )
+
+        metric = ReportMetric(
+            name=f"Initial Vignette Estimation - Beam {beam}",
+            description="These plots show the solar atlas fit used to estimate the initial, 1D spectral vignette "
+            "present in solar gain frames. The vignette signature is taken to be the fit continuum shown.",
+            metric_code=VispMetricCode.solar_first_vignette,
+            facet=self._format_facet(f"Beam {beam}"),
+            multi_plot_data=full_plot,
+        )
+
+        return metric.model_dump()
+
+    def build_final_vignette_metric(self, beam: int) -> dict:
+        """Build a ReportMetric showing the quality of the vignette correction on solar gain data."""
+        data = next(
+            self.read(
+                tags=[VispTag.quality(VispMetricCode.solar_final_vignette), VispTag.beam(beam)],
+                decoder=asdf_decoder,
+            )
+        )
+
+        wave_vec = data["output_wave_vec"].tolist()
+        median_spec = data["median_spec"].tolist()
+        low_deviation = data["low_deviation"]
+        high_deviation = data["high_deviation"]
+        diff = (high_deviation - low_deviation).tolist()
+        low_deviation = low_deviation.tolist()
+        high_deviation = high_deviation.tolist()
+
+        bounds_series = {
+            "Median solar signal": [wave_vec, median_spec],
+            "5th percentile bounds": [wave_vec, low_deviation],
+            "95th percentile bounds": [wave_vec, high_deviation],
+        }
+
+        bounds_plot_kwargs = {
+            "Median solar signal": {"ls": "-", "color": "k", "alpha": 0.8, "ms": 0, "zorder": 2.2},
+            "5th percentile bounds": {"color": "#1E317A", "ls": "-", "ms": 0, "zorder": 2.0},
+            "95th percentile bounds": {"ls": "-", "color": "#FAA61C", "ms": 0, "zorder": 2.1},
+        }
+
+        bounds_plot = Plot2D(
+            xlabel="Wavelength [nm]",
+            ylabel="Signal",
+            series_data=bounds_series,
+            plot_kwargs=bounds_plot_kwargs,
+            sort_series=False,
+        )
+
+        residuals_series = {"Residuals": [wave_vec, diff]}
+        residuals_plot_kwargs = {"Residuals": {"ls": "-", "color": "k", "ms": 0}}
+
+        y_min = np.nanpercentile(diff, 5)
+        y_max = np.nanpercentile(diff, 95)
+        y_range = y_max - y_min
+        y_min -= 0.1 * y_range
+        y_max += 0.1 * y_range
+        residuals_plot = Plot2D(
+            xlabel="Wavelength [nm]",
+            ylabel="95th - 5th percentile",
+            series_data=residuals_series,
+            plot_kwargs=residuals_plot_kwargs,
+            ylim=(y_min, y_max),
+        )
+
+        plot_list = [bounds_plot, residuals_plot]
+        height_ratios = [1.5, 1.0]
+
+        full_plot = VerticalMultiPanePlot2D(
+            top_to_bottom_plot_list=plot_list,
+            match_x_axes=True,
+            no_gap=True,
+            top_to_bottom_height_ratios=height_ratios,
+        )
+
+        metric = ReportMetric(
+            name=f"Final Vignette Estimation - Beam {beam}",
+            description="These plots show how well the full, 2D vignette signal was removed from solar gain frames. "
+            "The median solar signal shows a full spatial median of the vignette corrected solar gain; "
+            "this should be very close to the true solar spectrum incident on the DKIST optics. "
+            "The 5th and 9th percentile ranges show how stable this spectrum is along the spatial dimension "
+            "after removing the vignette signal.",
+            metric_code=VispMetricCode.solar_final_vignette,
+            facet=self._format_facet(f"Beam {beam}"),
+            multi_plot_data=full_plot,
+        )
+
+        return metric.model_dump()

dkist_processing_visp/tasks/lamp.py

@@ -1,6 +1,5 @@
 """ViSP lamp calibration task. See :doc:`this page </gain_correction>` for more information."""
 
-import numpy as np
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.fits import fits_array_decoder
 from dkist_processing_common.codecs.fits import fits_array_encoder
@@ -30,13 +29,14 @@ class LampCalibration(
 
     Parameters
     ----------
-    recipe_run_id : int
+    recipe_run_id
         id of the recipe run used to identify the workflow run this task is part of
-    workflow_name : str
+
+    workflow_name
         name of the workflow to which this instance of the task belongs
-    workflow_version : str
-        version of the workflow to which this instance of the task belongs
 
+    workflow_version
+        version of the workflow to which this instance of the task belongs
     """
 
     record_provenance = True
@@ -45,9 +45,11 @@ class LampCalibration(
         """
         For each beam.
 
-            - Gather input lamp gain and averaged dark arrays
-            - Calculate master lamp
-            - Write master lamp
+            - Normalize all input arrays by the number of frames per FPA
+            - Subtract the average dark frame corresponding to the matching readout exposure time
+            - Average all different readout exposure time arrays (if applicable)
+            - Interpolate over the hairlines
+            - Write final lamp gain to disk
             - Record quality metrics
 
         Returns
@@ -58,9 +60,11 @@ class LampCalibration(
         with self.telemetry_span(
             f"Generate lamp gains for {self.constants.num_beams} beams and {len(self.constants.lamp_readout_exp_times)} exposure times"
         ):
-            for readout_exp_time in self.constants.lamp_readout_exp_times:
-                for beam in range(1, self.constants.num_beams + 1):
-                    logger.info(f"Load dark for beam {beam}")
+            for beam in range(1, self.constants.num_beams + 1):
+                all_exp_time_arrays = []
+                for readout_exp_time in self.constants.lamp_readout_exp_times:
+                    apm_str = f"{beam = } and {readout_exp_time = }"
+                    logger.info(f"Load dark for beam {apm_str}")
                     dark_array = next(
                         self.read(
                             tags=VispTag.intermediate_frame_dark(
@@ -70,19 +74,44 @@ class LampCalibration(
                         )
                     )
 
-                    for state_num in range(
-                        1, self.constants.num_modstates + 1
-                    ):  # modulator states go from 1 to n
-                        logger.info(
-                            f"Calculating average lamp gain for beam {beam}, modulator state {state_num}"
+                    with self.telemetry_span(f"Computing gain for {apm_str}"):
+                        tags = [
+                            VispTag.input(),
+                            VispTag.frame(),
+                            VispTag.task_lamp_gain(),
+                            VispTag.readout_exp_time(readout_exp_time),
+                        ]
+                        input_lamp_gain_objs = self.read(
+                            tags=tags,
+                            decoder=fits_access_decoder,
+                            fits_access_class=VispL0FitsAccess,
+                        )
+
+                        readout_normalized_arrays = (
+                            self.beam_access_get_beam(o.data, beam=beam) / o.num_raw_frames_per_fpa
+                            for o in input_lamp_gain_objs
                         )
-                        self.compute_and_write_master_lamp_gain_for_modstate(
-                            modstate=state_num,
-                            dark_array=dark_array,
-                            beam=beam,
-                            readout_exp_time=readout_exp_time,
+                        averaged_gain_data = average_numpy_arrays(readout_normalized_arrays)
+
+                        dark_corrected_gain_data = next(
+                            subtract_array_from_arrays(averaged_gain_data, dark_array)
                         )
 
+                        all_exp_time_arrays.append(dark_corrected_gain_data)
+
+                avg_gain_array = average_numpy_arrays(all_exp_time_arrays)
+                filtered_gain_data = self.corrections_mask_hairlines(avg_gain_array)
+
+                with self.telemetry_span(f"Writing gain array for {apm_str}"):
+                    self.write(
+                        data=filtered_gain_data,
+                        tags=[
+                            VispTag.intermediate_frame(beam=beam),
+                            VispTag.task_lamp_gain(),
+                        ],
+                        encoder=fits_array_encoder,
+                    )
+
         with self.telemetry_span("Computing and logging quality metrics"):
             no_of_raw_lamp_frames: int = self.scratch.count_all(
                 tags=[
@@ -95,73 +124,3 @@ class LampCalibration(
             self.quality_store_task_type_counts(
                 task_type=TaskName.lamp_gain.value, total_frames=no_of_raw_lamp_frames
             )
-
-    def compute_and_write_master_lamp_gain_for_modstate(
-        self,
-        modstate: int,
-        dark_array: np.ndarray,
-        beam: int,
-        readout_exp_time: float,
-    ) -> None:
-        """
-        Compute and write master lamp gain for a given modstate and beam.
-
-        Generally the algorithm is:
-            1. Average input gain arrays
-            2. Subtract average dark to get the dark corrected gain data
-            3. Normalize each beam to unity mean
-            4. Write to disk
-
-        Parameters
-        ----------
-        modstate : int
-            The modulator state to calculate the master lamp gain for
-
-        dark_array : np.ndarray
-            The master dark to be subtracted from each lamp gain file
-
-        beam : int
-            The number of the beam
-
-        readout_exp_time : float
-            Exposure time of single readout
-
-        Returns
-        -------
-        None
-        """
-        apm_str = f"{beam = }, {modstate = }, and {readout_exp_time = }"
-        # Get the input lamp gain arrays
-        tags = [
-            VispTag.input(),
-            VispTag.frame(),
-            VispTag.task_lamp_gain(),
-            VispTag.modstate(modstate),
-            VispTag.readout_exp_time(readout_exp_time),
-        ]
-        input_lamp_gain_objs = self.read(
-            tags=tags, decoder=fits_access_decoder, fits_access_class=VispL0FitsAccess
-        )
-
-        with self.telemetry_span(f"Computing gain for {apm_str}"):
-
-            readout_normalized_arrays = (
-                self.beam_access_get_beam(o.data, beam=beam) / o.num_raw_frames_per_fpa
-                for o in input_lamp_gain_objs
-            )
-            averaged_gain_data = average_numpy_arrays(readout_normalized_arrays)
-
-            dark_corrected_gain_data = next(
-                subtract_array_from_arrays(averaged_gain_data, dark_array)
-            )
-            filtered_gain_data = self.corrections_mask_hairlines(dark_corrected_gain_data)
-
-        with self.telemetry_span(f"Writing gain array for {apm_str}"):
-            self.write(
-                data=filtered_gain_data,
-                tags=[
-                    VispTag.intermediate_frame(beam=beam, modstate=modstate),
-                    VispTag.task_lamp_gain(),
-                ],
-                encoder=fits_array_encoder,
-            )

dkist_processing_visp/tasks/parse.py

@@ -2,6 +2,7 @@
 
 from typing import TypeVar
 
+from dkist_processing_common.models.fits_access import MetadataKey
 from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_common.parsers.cs_step import CSStepFlower
@@ -15,6 +16,7 @@ from dkist_processing_common.parsers.time import ObsIpStartTimeBud
 from dkist_processing_common.parsers.time import ReadoutExpTimeFlower
 from dkist_processing_common.parsers.time import TaskExposureTimesBud
 from dkist_processing_common.parsers.time import TaskReadoutExpTimesBud
+from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
 from dkist_processing_common.parsers.unique_bud import UniqueBud
 from dkist_processing_common.parsers.wavelength import ObserveWavelengthBud
 from dkist_processing_common.tasks import ParseL0InputDataBase
@@ -29,6 +31,8 @@ from dkist_processing_visp.parsers.modulator_states import NumberModulatorStates
 from dkist_processing_visp.parsers.polarimeter_mode import PolarimeterModeBud
 from dkist_processing_visp.parsers.raster_step import RasterScanStepFlower
 from dkist_processing_visp.parsers.raster_step import TotalRasterStepsBud
+from dkist_processing_visp.parsers.spectrograph_configuration import IncidentLightAngleBud
+from dkist_processing_visp.parsers.spectrograph_configuration import ReflectedLightAngleBud
 from dkist_processing_visp.parsers.time import DarkReadoutExpTimePickyBud
 from dkist_processing_visp.parsers.time import NonDarkNonPolcalTaskReadoutExpTimesBud
 from dkist_processing_visp.parsers.visp_l0_fits_access import VispL0FitsAccess
@@ -74,6 +78,7 @@ class ParseL0VispInputData(ParseL0InputDataBase):
     def constant_buds(self) -> list[S]:
         """Add ViSP specific constants to common constants."""
         return super().constant_buds + [
+            UniqueBud(constant_name=VispBudName.arm_id.value, metadata_key=VispMetadataKey.arm_id),
             NumMapScansBud(),
             TotalRasterStepsBud(),
             NumCSStepBud(self.parameters.max_cs_step_time_sec),
@@ -84,6 +89,20 @@ class ParseL0VispInputData(ParseL0InputDataBase):
             RetarderNameBud(),
             NonDarkNonPolcalTaskReadoutExpTimesBud(),
             DarkReadoutExpTimePickyBud(),
+            IncidentLightAngleBud(),
+            ReflectedLightAngleBud(),
+            TaskUniqueBud(
+                constant_name=VispBudName.grating_constant_inverse_mm.value,
+                metadata_key=VispMetadataKey.grating_constant_inverse_mm,
+                ip_task_types=[TaskName.observe.value, TaskName.solar_gain.value],
+                task_type_parsing_function=parse_header_ip_task_with_gains,
+            ),
+            TaskUniqueBud(
+                constant_name=VispBudName.solar_gain_ip_start_time.value,
+                metadata_key=MetadataKey.ip_start_time,
+                ip_task_types=TaskName.solar_gain,
+                task_type_parsing_function=parse_header_ip_task_with_gains,
+            ),
             TaskExposureTimesBud(
                 stem_name=VispBudName.lamp_exposure_times.value,
                 ip_task_types=TaskName.lamp_gain.value,

dkist_processing_visp/tasks/science.py

@@ -170,7 +170,7 @@ class ScienceCalibration(
         """
         dark_dict = defaultdict(dict)
         background_dict = dict()
-        solar_dict = defaultdict(dict)
+        solar_dict = dict()
         angle_dict = dict()
         state_offset_dict = defaultdict(dict)
         spec_shift_dict = dict()
@@ -225,6 +225,18 @@ class ScienceCalibration(
                 )
             )
 
+            # Solar
+            #######
+            solar_dict[VispTag.beam(beam)] = next(
+                self.read(
+                    tags=[
+                        VispTag.intermediate_frame(beam=beam),
+                        VispTag.task_solar_gain(),
+                    ],
+                    decoder=fits_array_decoder,
+                )
+            )
+
             # Demod
             #######
             if self.constants.correct_for_polarization:
@@ -239,18 +251,6 @@ class ScienceCalibration(
                 )
 
             for modstate in range(1, self.constants.num_modstates + 1):
-                # Solar
-                #######
-                solar_dict[VispTag.beam(beam)][VispTag.modstate(modstate)] = next(
-                    self.read(
-                        tags=[
-                            VispTag.intermediate_frame(beam=beam, modstate=modstate),
-                            VispTag.task_solar_gain(),
-                        ],
-                        decoder=fits_array_decoder,
-                    )
-                )
-
                 # State Offset
                 ##############
                 state_offset_dict[VispTag.beam(beam)][VispTag.modstate(modstate)] = next(
@@ -586,7 +586,7 @@ class ScienceCalibration(
             VispTag.readout_exp_time(readout_exp_time)
         ]
         background_array = calibrations.background[VispTag.beam(beam)]
-        solar_gain_array = calibrations.solar_gain[VispTag.beam(beam)][VispTag.modstate(modstate)]
+        solar_gain_array = calibrations.solar_gain[VispTag.beam(beam)]
         angle = calibrations.angle[VispTag.beam(beam)]
         spec_shift = calibrations.spec_shift[VispTag.beam(beam)]
         state_offset = calibrations.state_offset[VispTag.beam(beam)][VispTag.modstate(modstate)]