dkist-processing-cryonirsp 1.3.4__py3-none-any.whl

dkist_processing_cryonirsp/tasks/assemble_movie.py

@@ -0,0 +1,202 @@
+"""CryoNIRSP-specific assemble movie task subclass."""
+import numpy as np
+from dkist_processing_common.codecs.fits import fits_array_decoder
+from dkist_processing_common.tasks import AssembleMovie
+from dkist_service_configuration.logging import logger
+from PIL import ImageDraw
+from PIL.ImageFont import FreeTypeFont
+
+from dkist_processing_cryonirsp.models.constants import CryonirspConstants
+from dkist_processing_cryonirsp.models.tags import CryonirspTag
+from dkist_processing_cryonirsp.parsers.cryonirsp_l0_fits_access import CryonirspL0FitsAccess
+from dkist_processing_cryonirsp.parsers.cryonirsp_l1_fits_access import CryonirspL1FitsAccess
+
+__all__ = ["AssembleCryonirspMovie", "SPAssembleCryonirspMovie"]
+
+
+class AssembleCryonirspMovieBase(AssembleMovie):
+    """
+    Assemble all CryoNIRSP movie frames (tagged with CryonirspTag.movie_frame()) into an mp4 movie file.
+
+    Subclassed from the AssembleMovie task in dkist_processing_common to add CryoNIRSP specific text overlays.
+
+
+    Parameters
+    ----------
+    recipe_run_id : int
+        id of the recipe run used to identify the workflow run this task is part of
+    workflow_name : str
+        name of the workflow to which this instance of the task belongs
+    workflow_version : str
+        version of the workflow to which this instance of the task belongs
+    """
+
+    MPL_COLOR_MAP = "gray"
+
+    def compute_frame_shape(self) -> tuple[int, int]:
+        """Dynamically set the dimensions of the movie based on L1 file shape."""
+        movie_frame_arrays = self.read(
+            tags=[CryonirspTag.movie_frame()], decoder=fits_array_decoder
+        )
+        random_frame = next(movie_frame_arrays)
+        raw_L1_shape = random_frame.shape
+        flipped_shape = raw_L1_shape[::-1]
+
+        standard_HD_num_pix = 1920 * 1080
+        frame_num_pix = np.prod(flipped_shape)
+        scale_factor = (
+            np.sqrt(standard_HD_num_pix / frame_num_pix)
+            if frame_num_pix > standard_HD_num_pix
+            else 1.0
+        )
+        scaled_shape = tuple(int(i * scale_factor) for i in flipped_shape)
+
+        return scaled_shape
+
+    def pre_run(self) -> None:
+        """Set the movie frame shape prior to running."""
+        super().pre_run()
+        frame_shape = self.compute_frame_shape()
+        logger.info(f"Setting movie shape to {frame_shape}")
+        self.MOVIE_FRAME_SHAPE = frame_shape
+
+    @property
+    def constants_model_class(self):
+        """Get CryoNIRSP constants."""
+        return CryonirspConstants
+
+    @property
+    def fits_parsing_class(self):
+        """Cryonirsp specific subclass of L1FitsAccess to use for reading images."""
+        return CryonirspL1FitsAccess
+
+    def write_overlay(self, draw: ImageDraw, fits_obj: CryonirspL0FitsAccess) -> None:
+        """
+        Mark each image with its instrument, observed wavelength, and observation time.
+
+        Parameters
+        ----------
+        draw
+            A PIL.ImageDraw object
+
+        fits_obj
+            A single movie "image", i.e., a single array tagged with CryonirspTag.movie_frame
+        """
+        self.write_line(
+            draw=draw,
+            text=f"INSTRUMENT: {self.constants.instrument}",
+            line=3,
+            column="right",
+            font=self.font_36,
+        )
+        self.write_line(
+            draw=draw,
+            text=f"WAVELENGTH: {fits_obj.wavelength} nm",
+            line=2,
+            column="right",
+            font=self.font_36,
+        )
+        self.write_line(
+            draw=draw,
+            text=f"OBS TIME: {fits_obj.time_obs}",
+            line=1,
+            column="right",
+            font=self.font_36,
+        )
+
+        if self.constants.correct_for_polarization:
+            self.write_line(draw=draw, text="Stokes-I", line=4, column="right", font=self.font_36)
+
+    def get_middle_line(self, draw: ImageDraw, text: str, font: FreeTypeFont) -> int:
+        """
+        Get the line number for the middle of the frame.
+
+        We need to compute this in real time because the frame size is dynamically based on the L1 file shape.
+        """
+        _, _, _, text_height = draw.textbbox(xy=(0, 0), text=text, font=font)
+        # See `write_line` in `dkist-processing-common` for why this is the expression.
+        line = (self.MOVIE_FRAME_SHAPE[1] // 2) / (self.TEXT_MARGIN_PX + text_height)
+        return line
+
+
+# See note below on `SPAssembleCryonirspMovie`
+class AssembleCryonirspMovie(AssembleCryonirspMovieBase):
+    """
+    Assemble all CryoNIRSP CI movie frames (tagged with CryonirspTag.movie_frame()) into an mp4 movie file.
+
+    Subclassed from the AssembleMovie task in dkist_processing_common to add CryoNIRSP specific text overlays.
+
+
+    Parameters
+    ----------
+    recipe_run_id : int
+        id of the recipe run used to identify the workflow run this task is part of
+    workflow_name : str
+        name of the workflow to which this instance of the task belongs
+    workflow_version : str
+        version of the workflow to which this instance of the task belongs
+
+
+    """
+
+    @property
+    def num_images(self) -> int:
+        """Total number of images in final movie.
+
+        Overloaded from `dkist-processing-common` because DSPS repeat does not correspond to map scan in CryoNIRSP
+        """
+        return self.constants.num_map_scans * self.constants.num_scan_steps
+
+    def tags_for_image_n(self, n: int) -> list[str]:
+        """Return tags that grab the n'th movie image.
+
+        Overloaded from `dkist-processing-common` because DSPS repeat does not correspond to map scan in CryoNIRSP
+        """
+        map_scan_num = n // self.constants.num_scan_steps + 1
+        scan_step = n % self.constants.num_scan_steps + 1
+
+        tags = [
+            CryonirspTag.map_scan(map_scan_num),
+            CryonirspTag.scan_step(scan_step),
+        ]
+        logger.info(f"AssembleMovie.tags_for_image_n: {tags =}")
+        return tags
+
+
+# NOTE:
+# This task isn't used right now because the SP movies need some better handling of the wavelength dimension.
+# For the time being both arms use the `MakeCryonirspMovieFrames` task above.
+# See PR #91 for more information.
+class SPAssembleCryonirspMovie(AssembleCryonirspMovieBase):
+    """
+    Assemble all CryoNIRSP SP movie frames (tagged with CryonirspTag.movie_frame()) into an mp4 movie file.
+
+    Subclassed from the AssembleMovie task in dkist_processing_common to add CryoNIRSP specific text overlays.
+
+
+    Parameters
+    ----------
+    recipe_run_id : int
+        id of the recipe run used to identify the workflow run this task is part of
+    workflow_name : str
+        name of the workflow to which this instance of the task belongs
+    workflow_version : str
+        version of the workflow to which this instance of the task belongs
+
+
+    """
+
+    @property
+    def num_images(self) -> int:
+        """Total number of images in final movie.
+
+        Overloaded from `dkist-processing-common` because DSPS repeat does not correspond to map scan in CryoNIRSP
+        """
+        return self.constants.num_map_scans
+
+    def tags_for_image_n(self, n: int) -> list[str]:
+        """Return tags that grab the n'th movie image.
+
+        Overloaded from `dkist-processing-common` because DSPS repeat does not correspond to map scan in CryoNIRSP
+        """
+        return [CryonirspTag.map_scan(n + 1)]

dkist_processing_cryonirsp/tasks/bad_pixel_map.py

@@ -0,0 +1,96 @@
+"""Task class for bad pixel map computation."""
+import numpy as np
+import scipy.ndimage as spnd
+from dkist_processing_math.statistics import average_numpy_arrays
+from dkist_service_configuration.logging import logger
+
+from dkist_processing_cryonirsp.models.tags import CryonirspTag
+from dkist_processing_cryonirsp.models.task_name import CryonirspTaskName
+from dkist_processing_cryonirsp.tasks.cryonirsp_base import CryonirspTaskBase
+
+__all__ = ["BadPixelMapCalibration"]
+
+
+class BadPixelMapCalibration(CryonirspTaskBase):
+    """
+    Task class for calculation of the bad pixel map for later use during calibration.
+
+    Parameters
+    ----------
+    recipe_run_id : int
+        id of the recipe run used to identify the workflow run this task is part of
+    workflow_name : str
+        name of the workflow to which this instance of the task belongs
+    workflow_version : str
+        version of the workflow to which this instance of the task belongs
+
+    """
+
+    record_provenance = True
+
+    def run(self):
+        """
+        Compute the bad pixel map by analyzing a set of solar gain images.
+
+        Steps:
+        1. Compute the average gain image
+        2. Smooth the array with a median filter
+        3. Calculate the difference between the smoothed and input arrays
+        4. Threshold the difference array on both ends to determine good and bad pixels
+        5. Save the bad pixel map as a fits file
+
+        Returns
+        -------
+        None
+
+        """
+        with self.apm_task_step(f"Compute average uncorrected solar gain image"):
+            average_solar_gain_array = self.compute_average_gain_array()
+
+        with self.apm_task_step(f"Compute the bad pixel map"):
+            with self.apm_processing_step("Smooth array with median filter"):
+                filter_size = self.parameters.bad_pixel_map_median_filter_size
+                filtered_array = spnd.median_filter(
+                    average_solar_gain_array,
+                    size=filter_size,
+                    mode="constant",
+                    cval=np.nanmedian(average_solar_gain_array),
+                )
+
+            with self.apm_processing_step("Identify bad pixels"):
+                thresh = self.parameters.bad_pixel_map_threshold_factor
+
+                diff = filtered_array - average_solar_gain_array
+                bad_pixel_map = np.array((np.abs(diff) > thresh * diff.std()), dtype=int)
+
+                # Find and fix any residual zeros that slipped through the bad pixel map.
+                zeros = np.where(average_solar_gain_array == 0.0)
+                bad_pixel_map[zeros] = 1
+
+        with self.apm_writing_step("Writing bad pixel map"):
+            self.intermediate_frame_write_arrays(
+                bad_pixel_map, task=CryonirspTaskName.bad_pixel_map.value
+            )
+
+    def compute_average_gain_array(self) -> np.ndarray:
+        """
+        Compute an average of uncorrected solar gain arrays.
+
+        We are computing the overall illumination pattern for one (CI) or both (SP) beams
+        simultaneously, so no dark correction is required and no beam splitting is used at
+        this point. Solar gain images are used because of their higher flux levels and they
+        more accurately represent the illuminated beam seen in solar images.
+
+        Returns
+        -------
+        The average gain array
+        """
+        lin_corr_gain_arrays = self.linearized_frame_full_array_generator(
+            tags=[
+                CryonirspTag.task_solar_gain(),
+                CryonirspTag.linearized(),
+                CryonirspTag.frame(),
+            ]
+        )
+        averaged_gain_data = average_numpy_arrays(lin_corr_gain_arrays)
+        return averaged_gain_data

dkist_processing_cryonirsp/tasks/beam_boundaries_base.py

@@ -0,0 +1,279 @@
+"""CryoNIRSP compute beam boundary task."""
+from abc import abstractmethod
+from dataclasses import dataclass
+
+import numpy as np
+from dkist_processing_math.statistics import average_numpy_arrays
+from dkist_service_configuration.logging import logger
+from largestinteriorrectangle import lir
+from skimage import filters
+from skimage.exposure import rescale_intensity
+from skimage.morphology import disk
+from skimage.util import img_as_ubyte
+
+from dkist_processing_cryonirsp.models.tags import CryonirspTag
+from dkist_processing_cryonirsp.tasks.cryonirsp_base import CryonirspTaskBase
+
+__all__ = ["BeamBoundariesCalibrationBase", "BeamBoundary"]
+
+
+@dataclass
+class BeamBoundary:
+    """Simple dataclass to hold boundary information for the illuminated portion of the beam array."""
+
+    y_min: int
+    y_max: int
+    x_min: int
+    x_max: int
+
+    @property
+    def y_slice(self):
+        """Return a slice object representing the illumination along the y-axis (numpy 0 axis)."""
+        return slice(self.y_min, self.y_max)
+
+    @property
+    def x_slice(self):
+        """Return a slice object representing the illumination along the x-axis (numpy 1 axis)."""
+        return slice(self.x_min, self.x_max)
+
+    @property
+    def beam_boundaries(self):
+        """Return a tuple containing the beam boundaries."""
+        return self.y_min, self.y_max, self.x_min, self.x_max
+
+    @property
+    def beam_boundaries_array(self):
+        """Return a tuple containing the beam boundaries."""
+        return np.array(self.beam_boundaries)
+
+
+class BeamBoundariesCalibrationBase(CryonirspTaskBase):
+    """
+    Task class for calculation of the beam boundaries for later use during calibration.
+
+    Parameters
+    ----------
+    recipe_run_id : int
+        id of the recipe run used to identify the workflow run this task is part of
+    workflow_name : str
+        name of the workflow to which this instance of the task belongs
+    workflow_version : str
+        version of the workflow to which this instance of the task belongs
+
+    """
+
+    record_provenance = True
+
+    def run(self):
+        """
+        Compute the beam boundaries by analyzing a set of solar gain images.
+
+        Steps:
+        1. Compute the average gain image
+        2. Correct any bad pixels
+        3. Smooth the image using a median filter
+        4. Split the beam along the horizontal axis (SP only)
+        5. Use a bimodal threshold filter to segment the image into illuminated and non-illuminated regions
+        6. Compute the boundaries of the illuminated region
+        7. Extract the illuminated portion of the beam images
+        8. Find the horizontal shift between the two images
+        9. Identify the boundaries of the overlap
+        10. Save the boundaries as a fits file (json?)
+
+        Returns
+        -------
+        None
+
+        """
+        # Step 1:
+        with self.apm_processing_step(f"Compute average solar gain image"):
+            average_solar_gain_array = self.compute_average_gain_array()
+
+        # Step 2:
+        with self.apm_task_step(f"Retrieve bad pixel map"):
+            bad_pixel_map = self.intermediate_frame_load_full_bad_pixel_map()
+            corrected_solar_gain_array = self.corrections_correct_bad_pixels(
+                average_solar_gain_array, bad_pixel_map
+            )
+
+        # Step 3
+        with self.apm_processing_step(f"Smooth the array to get good segmentation"):
+            smoothed_solar_gain_array = self.smooth_gain_array(corrected_solar_gain_array)
+
+        # Step 4
+        with self.apm_processing_step(f"Split the beam horizontally"):
+            split_beams = self.split_beams(smoothed_solar_gain_array)
+
+        # Step 5
+        with self.apm_processing_step(
+            f"Segment the beams into illuminated and non-illuminated pixels"
+        ):
+            segmented_beams = self.segment_arrays(split_beams)
+
+        # Step 6:
+        with self.apm_processing_step(
+            f"Compute the inscribed rectangular extents of the illuminated portions of the sensor"
+        ):
+            illuminated_boundaries = self.compute_boundaries_of_beam_illumination_regions(
+                segmented_beams
+            )
+
+        # Steps 7 - 9:
+        with self.apm_processing_step(f"Compute the boundaries of the illuminated beams"):
+            split_beams_float = [split_beam.astype(float) for split_beam in split_beams]
+            boundaries = self.compute_final_beam_boundaries(
+                split_beams_float, illuminated_boundaries
+            )
+
+        # Step 10:
+        with self.apm_writing_step("Writing beam boundaries"):
+            for beam, bounds in enumerate(boundaries, start=1):
+                self.intermediate_frame_write_arrays(
+                    bounds.beam_boundaries_array,
+                    task_tag=CryonirspTag.task_beam_boundaries(),
+                    beam=beam,
+                )
+
+    def compute_average_gain_array(self) -> np.ndarray:
+        """
+        Compute an average of uncorrected solar gain arrays.
+
+        We are computing the overall illumination pattern for both beams simultaneously,
+        so no dark correction is required and no beam splitting is used at this point.
+        Solar gain images are used because they have larger flux than the lamp gain images
+        and the lamp gain images do not have the same illumination pattern as the solar
+        gain images.
+
+        Returns
+        -------
+        The average gain array
+        """
+        lin_corr_gain_arrays = self.linearized_frame_full_array_generator(
+            tags=[
+                CryonirspTag.task_solar_gain(),
+                CryonirspTag.linearized(),
+                CryonirspTag.frame(),
+            ]
+        )
+        averaged_gain_data = average_numpy_arrays(lin_corr_gain_arrays)
+        return averaged_gain_data
+
+    def smooth_gain_array(self, array: np.ndarray) -> np.ndarray:
+        """
+        Smooth the input array with morphological filtering using a disk shape.
+
+        The array is smoothed to help eliminate artifacts in the image segmentation step.
+
+        Parameters
+        ----------
+        array
+            The input array to be smoothed
+
+        Returns
+        -------
+        The smoothed output array
+        """
+        # skimage.filters requires ubyte arrays and float->ubyte conversion only works with float in range [-1, 1]
+        norm_gain = img_as_ubyte(rescale_intensity(array, out_range=(0, 1.0)))
+
+        disk_size = self.parameters.beam_boundaries_smoothing_disk_size
+        norm_gain = filters.rank.median(norm_gain, disk(disk_size))
+        return norm_gain
+
+    @abstractmethod
+    def split_beams(self, input_array: np.ndarray) -> list[np.ndarray]:
+        """
+        Split the beams along the horizontal axis.
+
+        We use an abstract method so that the processing sequence is the same for both SP and CI
+        although the processing steps may be different.
+
+        Parameters
+        ----------
+        input_array
+            The array to be split
+
+        Returns
+        -------
+        [array, ...]
+            A list of arrays after the split
+        """
+        pass
+
+    @staticmethod
+    def segment_arrays(arrays: list[np.ndarray]) -> list[np.ndarray]:
+        """
+        Segment the arrays into illuminated (True) and non-illuminated (False) regions.
+
+        Parameters
+        ----------
+        arrays
+            The arrays to be segmented
+
+        Returns
+        -------
+        [array, ...]
+            The boolean segmented output arrays
+        """
+        segmented_arrays = []
+        for beam_num, beam_array in enumerate(arrays, start=1):
+            thresh = filters.threshold_minimum(beam_array)
+            logger.info(f"Segmentation threshold for beam {beam_num} = {thresh}")
+            segmented_arrays.append((beam_array > thresh).astype(bool))
+        return segmented_arrays
+
+    def compute_boundaries_of_beam_illumination_regions(
+        self, arrays: list[np.ndarray]
+    ) -> list[BeamBoundary]:
+        """
+        Compute the rectangular boundaries describing the illuminated regions of the beam images.
+
+        Parameters
+        ----------
+        arrays
+            A list of segmented boolean arrays over which the boundaries are to be computed
+
+        Returns
+        -------
+        [BeamBoundary, ...]
+        A list of the BeamBoundary objects representing the illuminated region(s) of the beam image(s)
+        """
+        boundaries = []
+        for beam, array in enumerate(arrays, start=1):
+            # Find the largest interior rectangle that inscribes the illuminated region for this beam
+            x_min, y_min, x_range, y_range = lir(array)
+            # Compute the new image bounds, the maximums are exclusive and can be used in slices
+            y_max = y_min + y_range
+            x_max = x_min + x_range
+
+            # Make sure all pixels are 1s
+            if not np.all(array[y_min:y_max, x_min:x_max]):
+                raise RuntimeError(
+                    f"Unable to compute illuminated image boundaries for beam {beam}"
+                )
+
+            boundaries.append(BeamBoundary(y_min, y_max, x_min, x_max))
+
+        return boundaries
+
+    @abstractmethod
+    def compute_final_beam_boundaries(
+        self,
+        smoothed_solar_gain_arrays: list[np.ndarray],
+        illuminated_boundaries: list[BeamBoundary],
+    ) -> list[BeamBoundary]:
+        """
+        Compute the final beam boundaries to be used when accessing individual beam images from the input dual-beam arrays.
+
+        Parameters
+        ----------
+        smoothed_solar_gain_arrays
+            Smoothed solar gain arrays to be used for coarsely aligning the individual beam images.
+        illuminated_boundaries
+            A list of BeamBoundary objects representing the illuminated regions of each beam image.
+
+        Returns
+        -------
+        A list of BeamBoundary objects describing the final beam boundaries for each beam.
+        """
+        pass

dkist_processing_cryonirsp/tasks/ci_beam_boundaries.py

@@ -0,0 +1,55 @@
+"""Cryonirsp CI beam boundaries task."""
+import largestinteriorrectangle as lir
+import numpy as np
+from dkist_service_configuration.logging import logger
+
+from dkist_processing_cryonirsp.tasks.beam_boundaries_base import BeamBoundariesCalibrationBase
+from dkist_processing_cryonirsp.tasks.beam_boundaries_base import BeamBoundary
+
+__all__ = ["CIBeamBoundariesCalibration"]
+
+
+class CIBeamBoundariesCalibration(BeamBoundariesCalibrationBase):
+    """Task class for calculation of the CI beam boundaries for later use during calibration."""
+
+    def split_beams(self, array: np.ndarray) -> list[np.ndarray]:
+        """
+        Return the input array, as there is no beam split for CI.
+
+        This is just a pass-through method to facilitate using the same processing
+        steps in the base run method.
+
+        Parameters
+        ----------
+        array
+            The input array
+
+        Returns
+        -------
+        [array]
+            The input array embedded in a list
+        """
+        return [array]
+
+    def compute_final_beam_boundaries(
+        self, smoothed_solar_gain_array: np.ndarray, illuminated_boundaries: list[BeamBoundary]
+    ):
+        """
+        Compute the final beam boundaries to be used when accessing CI beam images from the input dual-beam arrays.
+
+        Note: For CI, the illuminated beam boundaries ARE the final beam boundaries.
+        This method is simply a pass-through that allows both SP and CI to use the processing sequence.
+
+        Parameters
+        ----------
+        smoothed_solar_gain_array
+            The smoothed solar gain array for a single CI beam.
+        illuminated_boundaries
+            A list with a single BeamBoundary object for the illuminated CI beam.
+
+        Returns
+        -------
+        [BeamBoundary]
+            The CI BeamBoundary object embedded in a list.
+        """
+        return illuminated_boundaries