dkist-processing-vbi 1.17.4__py3-none-any.whl

dkist_processing_vbi/__init__.py

@@ -0,0 +1,9 @@
+"""Package providing support classes and methods used by all workflow tasks."""
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version
+
+try:
+    __version__ = version(distribution_name=__name__)
+except PackageNotFoundError:
+    # package is not installed
+    __version__ = "unknown"

dkist_processing_vbi/config.py

@@ -0,0 +1,12 @@
+"""Configuration for the dkist-processing-vbi package and the logging thereof."""
+from dkist_processing_common.config import DKISTProcessingCommonConfiguration
+
+
+class DKISTProcessingVBIConfigurations(DKISTProcessingCommonConfiguration):
+    """Configurations custom to the dkist-processing-vbi package."""
+
+    pass  # nothing custom yet
+
+
+dkist_processing_vbi_configurations = DKISTProcessingVBIConfigurations()
+dkist_processing_vbi_configurations.log_configurations()

dkist_processing_vbi/models/__init__.py

@@ -0,0 +1 @@
+"""init."""

dkist_processing_vbi/models/constants.py

@@ -0,0 +1,88 @@
+"""VBI additions to common constants."""
+from enum import Enum
+
+from dkist_processing_common.models.constants import ConstantsBase
+
+
+class VbiBudName(Enum):
+    """Names to be used in VBI buds."""
+
+    num_mosaics_repeats = "NUM_MOSAIC_REPEATS"
+    spatial_step_pattern = "SPATIAL_STEP_PATTERN"
+    num_spatial_steps = "NUM_SPATIAL_STEPS"
+    gain_exposure_times = "GAIN_EXPOSURE_TIMES"
+    observe_exposure_times = "OBSERVE_EXPOSURE_TIMES"
+
+
+class VbiConstants(ConstantsBase):
+    """VBI specific constants to add to the common constants."""
+
+    @property
+    def num_mosaic_repeats(self) -> int:
+        """Return the number of times the full mosaic is repeated."""
+        return self._db_dict[VbiBudName.num_mosaics_repeats.value]
+
+    @property
+    def mindices_of_mosaic_field_positions(self) -> list[tuple[int, int]]:
+        """
+        Define the mapping from mosaic field position to (MINDEX1, MINDEX2) tuple.
+
+        The mosaic fields are defined as:
+
+          BLUE     RED       SINGLE
+          1 2 3    1   2
+          4 5 6                5
+          7 8 9    3   4
+
+        Thus, the Nth index of this list gives the (MINDEX1, MINDEX2) tuple for the Nth mosaic field (a dummy 0th
+        element is provided so that the 1-indexing of the mosaic fields is preserved).
+        """
+        # fmt: off
+        # A dummy value goes in the 0th position because the mosaic fields are 1-indexed. This means we can
+        # simply slice this list with the mosaic field number.
+        blue_mosiac = ["index_placeholder",
+                       (1, 1), (1, 2), (1, 3),
+                       (2, 1), (2, 2), (2, 3),
+                       (3, 1), (3, 2), (3, 3),
+                       ]
+        red_mosiac = ["index_placeholder",
+                      (1, 1), (1, 2),
+                      (2, 1), (2, 2),
+                      ]
+        single = ["index_placeholder"] * 5 + [(1, 1)]
+        # fmt: on
+        match len(self.spatial_step_pattern):
+            case 1:
+                return single
+
+            case 4:
+                return red_mosiac
+
+            case 9:
+                return blue_mosiac
+
+            case _:
+                raise ValueError(
+                    f"Spatial step pattern {self.spatial_step_pattern} describes an unknown mosaic. Parsing should have caught this."
+                )
+
+    @property
+    def spatial_step_pattern(self) -> list[int]:
+        """Return a parsed list of the spatial step pattern used to scan the mosaic FOV."""
+        raw_list = self._db_dict[VbiBudName.spatial_step_pattern.value]
+        return [int(val) for val in raw_list.split(",")]
+
+    @property
+    def num_spatial_steps(self) -> int:
+        """Spatial steps in a raster."""
+        return self._db_dict[VbiBudName.num_spatial_steps.value]
+
+    @property
+    def gain_exposure_times(self) -> [float]:
+        """Exposure times of gain frames."""
+        return self._db_dict[VbiBudName.gain_exposure_times.value]
+
+    @property
+    def observe_exposure_times(self) -> [float]:
+        """Exposure times of observe frames."""
+        return self._db_dict[VbiBudName.observe_exposure_times.value]

dkist_processing_vbi/models/filter.py

@@ -0,0 +1,47 @@
+"""VBI filter list and tooling."""
+import astropy.units as u
+from dkist_processing_common.models.wavelength import WavelengthRange
+
+
+class Filter(WavelengthRange):
+    """
+    VBI filter data structure.
+
+    Parameters
+    ----------
+    name
+        The name of the filter
+    """
+
+    name: str
+
+
+VBI_FILTERS = [
+    Filter(name="VBI-Blue Ca II K", min=393.276 * u.nm, max=393.378 * u.nm),
+    Filter(name="VBI-Blue G-Band", min=430.301 * u.nm, max=430.789 * u.nm),
+    Filter(name="VBI-Blue Continuum", min=450.084 * u.nm, max=450.490 * u.nm),
+    Filter(name="VBI-Blue H-Beta", min=486.116 * u.nm, max=486.162 * u.nm),
+    Filter(name="VBI-Red H-alpha", min=656.258 * u.nm, max=656.306 * u.nm),
+    Filter(name="VBI-Red Continuum", min=668.202 * u.nm, max=668.644 * u.nm),
+    Filter(name="VBI-Red Ti O", min=705.545 * u.nm, max=706.133 * u.nm),
+    Filter(name="VBI-Red Fe IX", min=789.168 * u.nm, max=789.204 * u.nm),
+]
+
+
+def find_associated_filter(wavelength: u.Quantity) -> Filter:
+    """
+    Given a wavelength, find the Filter that contains that wavelength between its wavemin/wavemax.
+
+    Parameters
+    ----------
+    wavelength
+        The wavelength to use in the search
+
+    Returns
+    -------
+    A Filter object that contains the wavelength
+    """
+    matching_filters = [f for f in VBI_FILTERS if f.min <= wavelength <= f.max]
+    if len(matching_filters) == 1:
+        return matching_filters[0]
+    raise ValueError(f"Found {len(matching_filters)} matching filters when 1 was expected.")

dkist_processing_vbi/models/tags.py

@@ -0,0 +1,32 @@
+"""VBI tags."""
+from enum import Enum
+
+from dkist_processing_common.models.tags import Tag
+
+
+class VbiStemName(str, Enum):
+    """VBI specific tag stems."""
+
+    current_spatial_step = "STEP"
+    current_mosaic = "MOSAIC"
+
+
+class VbiTag(Tag):
+    """VBI specific tag formatting."""
+
+    @classmethod
+    def spatial_step(cls, step_num: int) -> str:
+        """
+        Tags by spatial step.
+
+        Parameters
+        ----------
+        step_num: int
+            The step number in the FOV
+        """
+        return cls.format_tag(VbiStemName.current_spatial_step, step_num)
+
+    @classmethod
+    def mosaic(cls, mosaic_num: int) -> str:
+        """Tags by mosaic number."""
+        return cls.format_tag(VbiStemName.current_mosaic, mosaic_num)

dkist_processing_vbi/parsers/__init__.py

@@ -0,0 +1 @@
+"""init."""

dkist_processing_vbi/parsers/mosaic_repeats.py

@@ -0,0 +1,167 @@
+"""Stems for organizing files based on their Mosaic repeat number."""
+from __future__ import annotations
+
+from abc import ABC
+from collections import defaultdict
+from functools import cached_property
+from typing import Type
+
+from astropy.time import Time
+from dkist_processing_common.models.flower_pot import SpilledDirt
+from dkist_processing_common.models.flower_pot import Stem
+from dkist_processing_common.models.task_name import TaskName
+
+from dkist_processing_vbi.models.constants import VbiBudName
+from dkist_processing_vbi.models.tags import VbiStemName
+from dkist_processing_vbi.parsers.vbi_l0_fits_access import VbiL0FitsAccess
+
+
+class SingleMosaicTile:
+    """
+    An object that uniquely defines a (mosaic_step, exp_num, time_obs) tuple from any number of mosaic repeats.
+
+    This is just a fancy tuple.
+
+    Basically, it just hashes the (mosaic_step, exp_num, time_obs) tuple so these objects can easily be compared.
+    Also uses the time_obs property so that multiple DSPS repeats of the same (mosaic_step, modstate) can be sorted.
+
+    This is just a fancy tuple.
+    """
+
+    def __init__(self, fits_obj: VbiL0FitsAccess):
+        """Read mosaic step, exp_num, and obs time information from a FitsAccess object."""
+        self.mosaic_step = fits_obj.current_spatial_step
+        self.exposure_num = fits_obj.current_mosaic_step_exp
+        self.date_obs = Time(fits_obj.time_obs)
+
+    def __repr__(self):
+        return f"SingleMosaicTile with {self.mosaic_step = }, {self.exposure_num = }, and {self.date_obs = }"
+
+    def __eq__(self, other: SingleMosaicTile) -> bool:
+        """Two frames are equal if they have the same (mosaic_step, exp_num, date_obs) tuple."""
+        if not isinstance(other, SingleMosaicTile):
+            raise TypeError(f"Cannot compare SingleMosaicTile with type {type(other)}")
+
+        for attr in ["mosaic_step", "exposure_num", "date_obs"]:
+            if getattr(self, attr) != getattr(other, attr):
+                return False
+
+        return True
+
+    def __lt__(self, other: SingleMosaicTile) -> bool:
+        """Only sort on date_obs."""
+        return self.date_obs < other.date_obs
+
+    def __hash__(self) -> int:
+        # Not strictly necessary, but does allow for using set() on these objects
+        return hash((self.mosaic_step, self.exposure_num, self.date_obs))
+
+
+class MosaicBase(Stem, ABC):
+    """Base class for Stems that use a dict of [int, Dict[int, SingleMosaicTile]] to parse mosaic tiles."""
+
+    # This only here so type-hinting of this complex dictionary will work.
+    key_to_petal_dict: dict[str, SingleMosaicTile]
+
+    @cached_property
+    def mosaic_tile_dict(self) -> dict[int, dict[int, list[SingleMosaicTile]]]:
+        """Nested dictionary that contains a SingleMosaicTile for each ingested frame.
+
+        Dictionary structure is [mosaic_step (int), Dict[exp_num (int), List[SingleMosaicTile]]
+        """
+        scan_step_dict = defaultdict(lambda: defaultdict(list))
+        for scan_step_obj in self.key_to_petal_dict.values():
+            scan_step_dict[scan_step_obj.mosaic_step][scan_step_obj.exposure_num].append(
+                scan_step_obj
+            )
+
+        return scan_step_dict
+
+    def setter(self, fits_obj: VbiL0FitsAccess) -> SingleMosaicTile | Type[SpilledDirt]:
+        """Ingest observe frames as SingleMosaicTile objects."""
+        if fits_obj.ip_task_type.casefold() != TaskName.observe.value.casefold():
+            return SpilledDirt
+        return SingleMosaicTile(fits_obj=fits_obj)
+
+
+class MosaicRepeatNumberFlower(MosaicBase):
+    """Flower for computing and assigning mosaic repeat numbers.
+
+    We can't use DKIST009 directly because subcycling in the instrument program task breaks the 1-to-1 mapping between
+    DSPS repeat and mosaic repeat
+    """
+
+    def __init__(self):
+        super().__init__(stem_name=VbiStemName.current_mosaic.value)
+
+    def getter(self, key: str) -> int:
+        """Compute the mosaic repeat number for a single frame.
+
+        A list of all the frames associated with a single (mosaic_step, exp_num) tuple is sorted by date and that list
+        is used to find the index of the frame in question.
+        """
+        mosaic_tile_obj = self.key_to_petal_dict[key]
+        all_obj_for_tile = sorted(
+            self.mosaic_tile_dict[mosaic_tile_obj.mosaic_step][mosaic_tile_obj.exposure_num]
+        )
+
+        num_files = all_obj_for_tile.count(mosaic_tile_obj)
+        if num_files > 1:
+            raise ValueError(
+                f"More than one file found for a single (mosaic_step, exp_num). Randomly selected example has {num_files} files."
+            )
+
+        # Here is where we decide that mosaic repeat numbers start at 1
+        return all_obj_for_tile.index(mosaic_tile_obj) + 1
+
+
+class TotalMosaicRepeatsBud(MosaicBase):
+    """Compute the total number of *complete* VBI Mosaics.
+
+    Note that an IP with only one camera position is still considered a "mosaic".
+
+    We can't use DKIST008 directly for two reasons:
+
+    1. Subcycling in the instrument program task breaks the 1-to-1 mapping between DSPS repeat and mosaic repeat
+
+    2. It's possible that the last repeat has an aborted mosaic. Instead, we return the number of completed mosaics found.
+    """
+
+    def __init__(self):
+        super().__init__(stem_name=VbiBudName.num_mosaics_repeats.value)
+
+    def getter(self, key: str) -> int:
+        """Compute the total number of mosaic repeats.
+
+        The number of mosaic repeats for every camera position are calculated and if a mosaic is incomplete,
+        it will not be included.
+        Assumes the incomplete mosaic is always the last one due to summit abort or cancellation.
+        """
+        # HOW THIS WORKS
+        ################
+        # self.mosaic_tile_dict conceptually looks like this:
+        # {mosaic_1:
+        #    {exp_1: [file1, file2],
+        #     exp_2: [file3, file4]},
+        #  mosaic_2:
+        #    {exp_1: [file5, file6],
+        #     exp_2: [file7, file7]}}
+        #
+        # We assume that each file for a (mosaic_step, exp_num) tuple is a different mosaic repeat
+        # (there are 2 repeats in the above example). So all we really need to do is find the lengths of all
+        # of the lists at the "exp_N" level.
+
+        # The k[0] assumes that a mosaic step has the same number of exposures for all DSPS repeats
+        repeats_per_mosaic_tile = [
+            k[0]
+            # The following list is the number of files found for each mosaic location for each exp_num
+            for k in [
+                # exp_dict is a dict of {exp_num: list(SingleMosaicTile)}
+                # so len(m) is the number of SingleMosaicTiles detected for each exp_num
+                [len(m) for m in exp_dict.values()]
+                for exp_dict in self.mosaic_tile_dict.values()
+            ]
+        ]
+        if min(repeats_per_mosaic_tile) + 1 < max(repeats_per_mosaic_tile):
+            raise ValueError("More than one incomplete mosaic exists in the data.")
+        return min(repeats_per_mosaic_tile)

dkist_processing_vbi/parsers/spatial_step_pattern.py

@@ -0,0 +1,43 @@
+"""Bud for checking that the spatial step pattern (VBISTPAT) matches expectations."""
+from dkist_processing_common.models.task_name import TaskName
+from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
+
+from dkist_processing_vbi.models.constants import VbiBudName
+
+
+class SpatialStepPatternBud(TaskUniqueBud):
+    """
+    Bud for checking and returning the VBI spatial step pattern.
+
+    This is just a `TaskUniqueBud` that performs the following checks on the unique value:
+
+      1. The step pattern must describe either a 1x1, 2x2, or 3x3 grid. This means it must have 1, 4, or 9 elements.
+
+      2. The step pattern cannot be empty.
+    """
+
+    def __init__(self):
+        super().__init__(
+            constant_name=VbiBudName.spatial_step_pattern.value,
+            metadata_key="spatial_step_pattern",
+            ip_task_type=TaskName.observe.value,
+        )
+
+    def getter(self, key):
+        """
+        Get a unique value and ensure it describes a valid step pattern.
+
+        See this Bud's class docstring for more information.
+        """
+        spatial_step_str = super().getter(key)
+        pos_list = spatial_step_str.split(",")
+        num_positions = len(pos_list)
+
+        # We need the check of pos_list[0] because `split` will always return a single element list
+        if num_positions not in [1, 4, 9] or pos_list[0] == "":
+            raise ValueError(
+                f'Spatial step pattern "{spatial_step_str}" does not represent either a 1x1, 2x2, or 3x3 mosaic. '
+                f"We don't know how to deal with this."
+            )
+
+        return spatial_step_str

dkist_processing_vbi/parsers/vbi_l0_fits_access.py

@@ -0,0 +1,36 @@
+"""VBI FITS access for L0 data."""
+from astropy.io import fits
+from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
+
+
+class VbiL0FitsAccess(L0FitsAccess):
+    """
+    Class to provide easy access to L0 headers.
+
+    i.e. instead of <VbiL0FitsAccess>.header['key'] this class lets us use <VbiL0FitsAccess>.key instead
+
+    Parameters
+    ----------
+    hdu :
+        Fits L0 header object
+
+    name : str
+        The name of the file that was loaded into this FitsAccess object
+
+    auto_squeeze : bool
+        When set to True, dimensions of length 1 will be removed from the array
+    """
+
+    def __init__(
+        self,
+        hdu: fits.ImageHDU | fits.PrimaryHDU | fits.CompImageHDU,
+        name: str | None = None,
+        auto_squeeze: bool = True,
+    ):
+        super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
+
+        self.spatial_step_pattern: str = self.header["VBISTPAT"]
+        self.number_of_spatial_steps: int = self.header.get("VBINSTP")
+        self.current_spatial_step: int = self.header.get("VBISTP")
+        self.number_of_exp_per_mosaic_step: int = self.header.get("VBINFRAM")
+        self.current_mosaic_step_exp: int = self.header.get("VBICFRAM")

dkist_processing_vbi/parsers/vbi_l1_fits_access.py

@@ -0,0 +1,33 @@
+"""VBI FITS access for L1 data."""
+from astropy.io import fits
+from dkist_processing_common.parsers.l1_fits_access import L1FitsAccess
+
+
+class VbiL1FitsAccess(L1FitsAccess):
+    """
+    Class to provide easy access to L1 headers.
+
+    i.e. instead of <VbiL1FitsAccess>.header['key'] this class lets us use <VbiL1FitsAccess>.key instead
+
+    Parameters
+    ----------
+    hdu :
+        Fits L1 header object
+
+    name : str
+        The name of the file that was loaded into this FitsAccess object
+
+    auto_squeeze : bool
+        When set to True, dimensions of length 1 will be removed from the array
+    """
+
+    def __init__(
+        self,
+        hdu: fits.ImageHDU | fits.PrimaryHDU | fits.CompImageHDU,
+        name: str | None = None,
+        auto_squeeze: bool = True,
+    ):
+        super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
+
+        self.number_of_spatial_steps: int = self.header.get("VBINSTP")
+        self.current_spatial_step: int = self.header.get("VBISTP")

dkist_processing_vbi/tasks/__init__.py

@@ -0,0 +1,10 @@
+"""init."""
+from dkist_processing_vbi.tasks.assemble_movie import *
+from dkist_processing_vbi.tasks.dark import *
+from dkist_processing_vbi.tasks.gain import *
+from dkist_processing_vbi.tasks.make_movie_frames import *
+from dkist_processing_vbi.tasks.parse import *
+from dkist_processing_vbi.tasks.process_summit_processed import *
+from dkist_processing_vbi.tasks.quality_metrics import *
+from dkist_processing_vbi.tasks.science import *
+from dkist_processing_vbi.tasks.write_l1 import *

dkist_processing_vbi/tasks/assemble_movie.py

@@ -0,0 +1,87 @@
+"""VBI-specific assemble movie task subclass."""
+from typing import Type
+
+import numpy as np
+from astropy.visualization import ZScaleInterval
+from dkist_processing_common.models.constants import ConstantsBase
+from dkist_processing_common.tasks import AssembleMovie
+from matplotlib import colormaps
+from PIL import ImageDraw
+
+from dkist_processing_vbi.models.constants import VbiConstants
+from dkist_processing_vbi.models.tags import VbiTag
+from dkist_processing_vbi.parsers.vbi_l1_fits_access import VbiL1FitsAccess
+
+__all__ = ["AssembleVbiMovie"]
+
+
+class AssembleVbiMovie(AssembleMovie):
+    """
+    Class for assembling pre-made movie frames (as FITS/numpy) into an mp4 movie file.
+
+    Subclassed from the AssembleMovie task in dkist_processing_common to add VBI specific text overlays.
+
+    Parameters
+    ----------
+    recipe_run_id : int
+        id of the recipe run used to identify the workflow run this task is part of
+    workflow_name : str
+        name of the workflow to which this instance of the task belongs
+    workflow_version : str
+        version of the workflow to which this instance of the task belongs
+    """
+
+    # So tab completion shows all the ViSP constants
+    constants: VbiConstants
+
+    @property
+    def constants_model_class(self) -> Type[ConstantsBase]:
+        """Class used to access constant database."""
+        return VbiConstants
+
+    @property
+    def fits_parsing_class(self):
+        """VBI specific subclass of L1FitsAccess to use for reading images."""
+        return VbiL1FitsAccess
+
+    @property
+    def num_images(self) -> int:
+        """Total number of images in final movie."""
+        return self.constants.num_mosaic_repeats
+
+    def tags_for_image_n(self, n: int) -> list[str]:
+        """Return the tags needed to find image n."""
+        return [VbiTag.mosaic(n + 1)]
+
+    def apply_colormap(self, array: np.ndarray) -> np.ndarray:
+        """
+        Convert floats to RGB colors using the ZScale normalization scheme.
+
+        Parameters
+        ----------
+        array : np.ndarray
+            data to convert
+        """
+        color_mapper = colormaps.get_cmap(self.MPL_COLOR_MAP)
+        scaled_array = ZScaleInterval()(array)
+        return color_mapper(scaled_array, bytes=True)[
+            :, :, :-1
+        ]  # Drop the last (alpha) color dimension
+
+    def write_overlay(self, draw: ImageDraw, fits_obj: VbiL1FitsAccess) -> None:
+        """
+        Mark each image with it's instrument, observed wavelength, and observation time.
+
+        Parameters
+        ----------
+        draw
+            A simple 2D drawing function for PIL images
+
+        fits_obj
+            A single movie "image", i.e., a single array tagged with VBITag.movie_frame
+        """
+        self.write_line(
+            draw, f"INSTRUMENT: {self.constants.instrument}", 3, "right", font=self.font_18
+        )
+        self.write_line(draw, f"WAVELENGTH: {fits_obj.wavelength}", 2, "right", font=self.font_15)
+        self.write_line(draw, f"DATE OBS: {fits_obj.time_obs}", 1, "right", font=self.font_15)