dkist-processing-vbi 1.34.8__tar.gz → 2.0.0__tar.gz

{dkist_processing_vbi-1.34.8 → dkist_processing_vbi-2.0.0}/CHANGELOG.rst

@@ -1,3 +1,14 @@
+v2.0.0 (2026-09-03)
+===================
+
+Features
+--------
+
+- Distinguish large, edge-connected gain occlusions from detector bad pixels, assigning occluded pixels a neutral gain of 1.0 and excluding them from bad-pixel correction. (`#231 <https://bitbucket.org/dkistdc/dkist-processing-vbi/pull-requests/231>`__)
+- Update VBI movies to use percentile-clipped linear intensity scaling. (`#231 <https://bitbucket.org/dkistdc/dkist-processing-vbi/pull-requests/231>`__)
+- Update VBI movies to display the observation's scanning mode. (`#231 <https://bitbucket.org/dkistdc/dkist-processing-vbi/pull-requests/231>`__)
+
+
 v1.34.8 (2026-08-25)
 ====================
 

{dkist_processing_vbi-1.34.8 → dkist_processing_vbi-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-vbi
-Version: 1.34.8
+Version: 2.0.0
 Summary: Processing code for the VBI instrument on DKIST
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -80,7 +80,7 @@ Requires-Dist: alembic==1.19.1; extra == "frozen"
 Requires-Dist: amqp==5.3.1; extra == "frozen"
 Requires-Dist: annotated-doc==0.0.5; extra == "frozen"
 Requires-Dist: annotated-types==0.8.0; extra == "frozen"
-Requires-Dist: anyio==4.14.2; extra == "frozen"
+Requires-Dist: anyio==4.15.0; extra == "frozen"
 Requires-Dist: apache-airflow==3.3.1; extra == "frozen"
 Requires-Dist: apache-airflow-core==3.3.1; extra == "frozen"
 Requires-Dist: apache-airflow-providers-celery==3.23.1; extra == "frozen"
@@ -101,7 +101,7 @@ Requires-Dist: asdf_standard==1.5.0; extra == "frozen"
 Requires-Dist: asgiref==3.12.1; extra == "frozen"
 Requires-Dist: asteval==1.0.10; extra == "frozen"
 Requires-Dist: astropy==7.0.2; extra == "frozen"
-Requires-Dist: astropy-iers-data==0.2026.8.24.0.24.29; extra == "frozen"
+Requires-Dist: astropy-iers-data==0.2026.8.31.0.57.9; extra == "frozen"
 Requires-Dist: astropy_healpix==2.0.1; extra == "frozen"
 Requires-Dist: asttokens==3.0.2; extra == "frozen"
 Requires-Dist: async-lru==2.3.0; extra == "frozen"
@@ -111,15 +111,15 @@ Requires-Dist: babel==2.18.0; extra == "frozen"
 Requires-Dist: beautifulsoup4==4.15.0; extra == "frozen"
 Requires-Dist: billiard==4.2.4; extra == "frozen"
 Requires-Dist: bleach==6.4.0; extra == "frozen"
-Requires-Dist: boto3==1.43.79; extra == "frozen"
-Requires-Dist: botocore==1.43.79; extra == "frozen"
-Requires-Dist: cachetools==7.1.7; extra == "frozen"
+Requires-Dist: boto3==1.43.88; extra == "frozen"
+Requires-Dist: botocore==1.43.88; extra == "frozen"
+Requires-Dist: cachetools==7.1.8; extra == "frozen"
 Requires-Dist: cadwyn==7.0.0; extra == "frozen"
 Requires-Dist: celery==5.6.3; extra == "frozen"
 Requires-Dist: certifi==2026.7.22; extra == "frozen"
 Requires-Dist: cffi==2.1.1; extra == "frozen"
 Requires-Dist: charset-normalizer==3.5.1; extra == "frozen"
-Requires-Dist: click==8.4.2; extra == "frozen"
+Requires-Dist: click==8.5.0; extra == "frozen"
 Requires-Dist: click-didyoumean==0.3.1; extra == "frozen"
 Requires-Dist: click-plugins==1.1.1.2; extra == "frozen"
 Requires-Dist: click-repl==0.3.0; extra == "frozen"
@@ -129,7 +129,7 @@ Requires-Dist: comm==0.2.3; extra == "frozen"
 Requires-Dist: contourpy==1.3.3; extra == "frozen"
 Requires-Dist: cron_descriptor==2.1.0; extra == "frozen"
 Requires-Dist: croniter==6.2.4; extra == "frozen"
-Requires-Dist: cryptography==50.0.0; extra == "frozen"
+Requires-Dist: cryptography==50.0.1; extra == "frozen"
 Requires-Dist: cycler==0.12.1; extra == "frozen"
 Requires-Dist: dask==2026.8.0; extra == "frozen"
 Requires-Dist: dask-image==2026.5.0; extra == "frozen"
@@ -142,7 +142,7 @@ Requires-Dist: dkist-processing-common==14.4.0; extra == "frozen"
 Requires-Dist: dkist-processing-core==7.5.0; extra == "frozen"
 Requires-Dist: dkist-processing-math==2.2.3; extra == "frozen"
 Requires-Dist: dkist-processing-pac==3.1.2; extra == "frozen"
-Requires-Dist: dkist-processing-vbi==1.34.8; extra == "frozen"
+Requires-Dist: dkist-processing-vbi==2.0.0; extra == "frozen"
 Requires-Dist: dkist-service-configuration==4.3.0; extra == "frozen"
 Requires-Dist: dkist-spectral-lines==3.0.1; extra == "frozen"
 Requires-Dist: dkist_fits_specifications==4.28.1; extra == "frozen"
@@ -154,17 +154,17 @@ Requires-Dist: fastapi==0.136.3; extra == "frozen"
 Requires-Dist: fastapi-cli==0.0.32; extra == "frozen"
 Requires-Dist: fastjsonschema==2.22.2; extra == "frozen"
 Requires-Dist: flower==2.1.0; extra == "frozen"
-Requires-Dist: fonttools==4.63.0; extra == "frozen"
+Requires-Dist: fonttools==4.64.0; extra == "frozen"
 Requires-Dist: fqdn==1.5.1; extra == "frozen"
 Requires-Dist: frozenlist==1.8.0; extra == "frozen"
 Requires-Dist: fsspec==2026.7.0; extra == "frozen"
 Requires-Dist: globus-sdk==4.9.0; extra == "frozen"
 Requires-Dist: google-crc32c==1.8.0; extra == "frozen"
-Requires-Dist: googleapis-common-protos==1.75.1; extra == "frozen"
+Requires-Dist: googleapis-common-protos==1.75.2; extra == "frozen"
 Requires-Dist: gqlclient==2.0.0; extra == "frozen"
 Requires-Dist: greenback==1.3.0; extra == "frozen"
 Requires-Dist: greenlet==3.5.5; extra == "frozen"
-Requires-Dist: grpcio==1.83.0; extra == "frozen"
+Requires-Dist: grpcio==1.83.1; extra == "frozen"
 Requires-Dist: h11==0.16.0; extra == "frozen"
 Requires-Dist: httpcore==1.0.9; extra == "frozen"
 Requires-Dist: httptools==0.8.0; extra == "frozen"
@@ -172,9 +172,9 @@ Requires-Dist: httpx==0.28.1; extra == "frozen"
 Requires-Dist: humanize==4.16.0; extra == "frozen"
 Requires-Dist: idna==3.19; extra == "frozen"
 Requires-Dist: imageio-ffmpeg==0.6.0; extra == "frozen"
-Requires-Dist: importlib_metadata==9.0.0; extra == "frozen"
+Requires-Dist: importlib_metadata==9.0.1; extra == "frozen"
 Requires-Dist: ipykernel==7.3.0; extra == "frozen"
-Requires-Dist: ipython==9.16.1; extra == "frozen"
+Requires-Dist: ipython==9.17.1; extra == "frozen"
 Requires-Dist: ipython_pygments_lexers==1.1.1; extra == "frozen"
 Requires-Dist: isoduration==20.11.0; extra == "frozen"
 Requires-Dist: itsdangerous==2.2.0; extra == "frozen"
@@ -187,20 +187,20 @@ Requires-Dist: jsonschema-specifications==2025.9.1; extra == "frozen"
 Requires-Dist: jupyter-events==0.12.1; extra == "frozen"
 Requires-Dist: jupyter-lsp==2.3.1; extra == "frozen"
 Requires-Dist: jupyter_builder==1.2.2; extra == "frozen"
-Requires-Dist: jupyter_client==8.9.1; extra == "frozen"
+Requires-Dist: jupyter_client==8.10.0; extra == "frozen"
 Requires-Dist: jupyter_core==5.9.1; extra == "frozen"
-Requires-Dist: jupyter_server==2.20.0; extra == "frozen"
+Requires-Dist: jupyter_server==2.21.0; extra == "frozen"
 Requires-Dist: jupyter_server_terminals==0.5.4; extra == "frozen"
 Requires-Dist: jupyterlab==4.6.3; extra == "frozen"
 Requires-Dist: jupyterlab_pygments==0.3.0; extra == "frozen"
 Requires-Dist: jupyterlab_server==2.28.0; extra == "frozen"
-Requires-Dist: kiwisolver==1.5.0; extra == "frozen"
+Requires-Dist: kiwisolver==1.5.1; extra == "frozen"
 Requires-Dist: kombu==5.6.2; extra == "frozen"
 Requires-Dist: lark==1.3.1; extra == "frozen"
 Requires-Dist: lazy-loader==0.5; extra == "frozen"
 Requires-Dist: lazy-object-proxy==1.12.0; extra == "frozen"
 Requires-Dist: libcst==1.9.0; extra == "frozen"
-Requires-Dist: linkify-it-py==2.1.1; extra == "frozen"
+Requires-Dist: linkify-it-py==2.2.0; extra == "frozen"
 Requires-Dist: lmfit==1.3.4; extra == "frozen"
 Requires-Dist: locket==1.0.0; extra == "frozen"
 Requires-Dist: lockfile==0.12.2; extra == "frozen"
@@ -264,27 +264,27 @@ Requires-Dist: pexpect==4.9.0; extra == "frozen"
 Requires-Dist: pika==1.4.4; extra == "frozen"
 Requires-Dist: pillow==10.4.0; extra == "frozen"
 Requires-Dist: pip==26.2.1; extra == "frozen"
-Requires-Dist: platformdirs==4.11.4; extra == "frozen"
+Requires-Dist: platformdirs==4.11.7; extra == "frozen"
 Requires-Dist: pluggy==1.6.0; extra == "frozen"
 Requires-Dist: pooch==1.9.0; extra == "frozen"
 Requires-Dist: proglog==0.1.12; extra == "frozen"
 Requires-Dist: prometheus_client==0.26.0; extra == "frozen"
 Requires-Dist: prompt_toolkit==3.0.53; extra == "frozen"
 Requires-Dist: propcache==0.5.2; extra == "frozen"
-Requires-Dist: protobuf==7.36.0; extra == "frozen"
+Requires-Dist: protobuf==7.36.1; extra == "frozen"
 Requires-Dist: psutil==7.2.2; extra == "frozen"
-Requires-Dist: psycopg==3.3.4; extra == "frozen"
-Requires-Dist: psycopg-binary==3.3.4; extra == "frozen"
+Requires-Dist: psycopg==3.3.5; extra == "frozen"
+Requires-Dist: psycopg-binary==3.3.5; extra == "frozen"
 Requires-Dist: psycopg2-binary==2.9.12; extra == "frozen"
 Requires-Dist: ptyprocess==0.7.0; extra == "frozen"
 Requires-Dist: pure_eval==0.2.3; extra == "frozen"
 Requires-Dist: pycparser==3.0; extra == "frozen"
-Requires-Dist: pydantic==2.13.4; extra == "frozen"
+Requires-Dist: pydantic==2.13.5; extra == "frozen"
 Requires-Dist: pydantic-extra-types==2.11.1; extra == "frozen"
 Requires-Dist: pydantic-settings==2.15.0; extra == "frozen"
-Requires-Dist: pydantic_core==2.46.4; extra == "frozen"
+Requires-Dist: pydantic_core==2.46.5; extra == "frozen"
 Requires-Dist: pyerfa==2.0.1.5; extra == "frozen"
-Requires-Dist: pygtrie==2.5.0; extra == "frozen"
+Requires-Dist: pygtrie==2.6.1; extra == "frozen"
 Requires-Dist: pyparsing==3.3.2; extra == "frozen"
 Requires-Dist: python-daemon==3.1.2; extra == "frozen"
 Requires-Dist: python-dateutil==2.9.0.post0; extra == "frozen"
@@ -335,12 +335,11 @@ Requires-Dist: toolz==1.1.0; extra == "frozen"
 Requires-Dist: tornado==6.5.8; extra == "frozen"
 Requires-Dist: tqdm==4.70.0; extra == "frozen"
 Requires-Dist: traitlets==5.16.1; extra == "frozen"
-Requires-Dist: typer==0.27.1; extra == "frozen"
+Requires-Dist: typer==0.27.2; extra == "frozen"
 Requires-Dist: typing-inspection==0.4.4; extra == "frozen"
 Requires-Dist: typing_extensions==4.16.0; extra == "frozen"
 Requires-Dist: tzdata==2026.3; extra == "frozen"
 Requires-Dist: tzlocal==5.4.4; extra == "frozen"
-Requires-Dist: uc-micro-py==2.0.0; extra == "frozen"
 Requires-Dist: uncertainties==3.2.3; extra == "frozen"
 Requires-Dist: universal_pathlib==0.3.10; extra == "frozen"
 Requires-Dist: uri-template==1.3.0; extra == "frozen"
@@ -351,13 +350,13 @@ Requires-Dist: uvloop==0.22.1; extra == "frozen"
 Requires-Dist: vine==5.1.0; extra == "frozen"
 Requires-Dist: voluptuous==0.16.0; extra == "frozen"
 Requires-Dist: watchfiles==1.2.0; extra == "frozen"
-Requires-Dist: wcwidth==0.8.2; extra == "frozen"
+Requires-Dist: wcwidth==0.8.3; extra == "frozen"
 Requires-Dist: webcolors==25.10.0; extra == "frozen"
 Requires-Dist: webencodings==0.6.1; extra == "frozen"
-Requires-Dist: websocket-client==1.9.0; extra == "frozen"
-Requires-Dist: websockets==17.0.1; extra == "frozen"
+Requires-Dist: websocket-client==1.9.2; extra == "frozen"
+Requires-Dist: websockets==17.1; extra == "frozen"
 Requires-Dist: wirerope==1.0.0; extra == "frozen"
-Requires-Dist: wrapt==2.3.0; extra == "frozen"
+Requires-Dist: wrapt==2.4.0; extra == "frozen"
 Requires-Dist: yamale==6.1.0; extra == "frozen"
 Requires-Dist: yarl==1.24.5; extra == "frozen"
 Requires-Dist: zarr==3.3.0; extra == "frozen"

dkist_processing_vbi-2.0.0/SCIENCE_CHANGELOG.rst

@@ -0,0 +1,18 @@
+v2.0.0 (2026-09-03)
+===================
+
+
+
+
+- Large, edge-connected regions identified as bad in a gain frame can now be classified as observation-related occlusions using the configurable ``vbi_occlusion_minimum_connected_pixels`` threshold.
+  Occluded pixels are excluded from full-field gain normalization and the bad-pixel map, then assigned a neutral gain of 1.0.
+  This prevents unilluminated detector regions from being interpolated as if they were defective camera pixels and avoids invalid values when a correction neighborhood would otherwise contain only flagged pixels. (`#231 <https://bitbucket.org/dkistdc/dkist-processing-vbi/pull-requests/231>`__)
+
+
+v1.1.0 (2022-12-15)
+===================
+
+
+
+
+- Don't re-compress already compressed data that are processed at the summit. This ensures L1 files served by the Data Center are *exactly* the same as those processed on the summit. (`#39 <https://bitbucket.org/dkistdc/dkist-processing-vbi/pull-requests/39>`__)

{dkist_processing_vbi-1.34.8 → dkist_processing_vbi-2.0.0}/dkist_processing_vbi/models/parameters.py

@@ -16,6 +16,11 @@ class VbiParameters(ParameterBase):
         """Return the number of standard deviations used to identify bad pixels."""
         return self._find_most_recent_past_value("vbi_bad_pixel_standard_deviations")
 
+    @property
+    def occlusion_minimum_connected_pixels(self) -> int:
+        """Return the minimum edge-connected area classified as an occlusion."""
+        return self._find_most_recent_past_value("vbi_occlusion_minimum_connected_pixels")
+
     @property
     def movie_intensity_clipping_percentile(self) -> float:
         """Percentile to clip the movie intensity at."""

{dkist_processing_vbi-1.34.8 → dkist_processing_vbi-2.0.0}/dkist_processing_vbi/tasks/assemble_movie.py

@@ -208,7 +208,7 @@ class AssembleVbiMovie(VbiTaskBase):
 
     def apply_colormap(self, array: np.ndarray) -> np.ndarray:
         """
-        Convert floats to RGB colors using the ZScale normalization scheme.
+        Convert floats to RGB colors using a linear normalization scheme.
 
         Parameters
         ----------
@@ -217,13 +217,23 @@ class AssembleVbiMovie(VbiTaskBase):
         """
         # Clip the top and bottom 0.5% of data values to improve the contrast
         movie_intensity_clipping_percentile = self.parameters.movie_intensity_clipping_percentile
-        vmin, vmax = np.nanpercentile(
-            array, (movie_intensity_clipping_percentile, 100 - movie_intensity_clipping_percentile)
-        )
-        array = np.clip(a=array, a_min=vmin, a_max=vmax)
+        finite = np.isfinite(array)
+        scaled_array = np.full(array.shape, np.nan)
+        if np.any(finite):
+            vmin, vmax = np.percentile(
+                array[finite],
+                (movie_intensity_clipping_percentile, 100 - movie_intensity_clipping_percentile),
+            )
+            if vmax > vmin:
+                scaled_array[finite] = np.clip(
+                    a=(array[finite] - vmin) / (vmax - vmin),
+                    a_min=0,
+                    a_max=1,
+                )
+            else:
+                scaled_array[finite] = 0.0
 
         color_mapper = colormaps.get_cmap("gray").with_extremes(bad="white")
-        scaled_array = ZScaleInterval()(array)
         return color_mapper(scaled_array, bytes=True)[
             :, :, :-1
         ]  # Drop the last (alpha) color dimension
@@ -436,8 +446,11 @@ class AssembleVbiMovie(VbiTaskBase):
         last_frame = self.movie_frames[-1]
         obs_part_id = self.metadata_store_input_dataset_observe_frames.inputDatasetPartId
         product_id = WriteL1Frame.compute_product_id(obs_part_id, "L1")
+        scanning_mode = (
+            "Center only" if first_frame.number_of_spatial_steps == 1 else "Field Scanning"
+        )
         return (
-            f"Instrument: {self.instrument_string} -- Wavelength: {self.constants.wavelength:.3f} nm\n"
+            f"Instrument: {self.instrument_string} -- Wavelength: {self.constants.wavelength:.3f} nm -- Scanning Mode: {scanning_mode}\n"
             f"Product ID: {product_id} --- Experiment ID: {self.constants.experiment_id}\n"
             f"Start Time: {first_frame.time_obs} --- End Time: {last_frame.time_obs}"
         )

{dkist_processing_vbi-1.34.8 → dkist_processing_vbi-2.0.0}/dkist_processing_vbi/tasks/gain.py

@@ -8,6 +8,7 @@ from dkist_processing_common.tasks.mixin.quality import QualityMixin
 from dkist_processing_math.arithmetic import subtract_array_from_arrays
 from dkist_processing_math.statistics import average_numpy_arrays
 from dkist_service_configuration.logging import logger
+from scipy import ndimage
 
 from dkist_processing_vbi.models.tags import VbiTag
 from dkist_processing_vbi.tasks.vbi_base import VbiTaskBase
@@ -52,8 +53,9 @@ class GainCalibration(VbiTaskBase, QualityMixin):
 
         # We'll just stuff the un-normalized arrays in this dictionary to avoid dealing with tags, io, etc.
         # This is OK (tm) because this will be, at most, 9 4k x 4k arrays. This is a lot (~1G), but not too much.
-        step_gain_dict: dict = {}
-        step_bad_pixel_map_dict: dict = {}
+        step_gain_dict: dict[int, np.ndarray] = {}
+        step_bad_pixel_map_dict: dict[int, np.ndarray] = {}
+        step_occlusion_mask_dict: dict[int, np.ndarray] = {}
 
         with self.telemetry_span(
             f"Collecting and reducing gain arrays from {self.constants.num_spatial_steps} steps and {len(self.constants.solar_gain_exposure_times)} exp times",
@@ -102,10 +104,13 @@ class GainCalibration(VbiTaskBase, QualityMixin):
                         averaged_gain_array=averaged_gain_array,
                         dark_subtracted_gain_array=dark_subtracted_gain_array,
                     )
+                    occlusion_mask = self.identify_occlusion_pixels(bad_pixel_map)
+                    step_occlusion_mask_dict[step] = occlusion_mask
+                    bad_pixel_map = bad_pixel_map & ~occlusion_mask
                     step_bad_pixel_map_dict[step] = bad_pixel_map
 
                     logger.info(f"Recording processed gain image for {step = }")
-                    usable_pixel_mask = ~bad_pixel_map
+                    usable_pixel_mask = ~(bad_pixel_map | occlusion_mask)
                     self.total_usable_pix[step] = np.count_nonzero(usable_pixel_mask)
                     self.total_counts[step] = np.sum(dark_subtracted_gain_array[usable_pixel_mask])
                     step_gain_dict[step] = dark_subtracted_gain_array
@@ -113,6 +118,12 @@ class GainCalibration(VbiTaskBase, QualityMixin):
         with self.telemetry_span("normalizing gain arrays"):
             normalized_array_dict = self.normalize_fov(step_gain_dict)
 
+        with self.telemetry_span("neutralizing occluded gain values"):
+            normalized_array_dict = self.neutralize_occlusion_gain_values(
+                gain_array_dict=normalized_array_dict,
+                step_occlusion_mask_dict=step_occlusion_mask_dict,
+            )
+
         with self.telemetry_span("replacing invalid inverse-gain values"):
             normalized_array_dict = self.replace_invalid_gain_values(normalized_array_dict)
 
@@ -160,6 +171,49 @@ class GainCalibration(VbiTaskBase, QualityMixin):
 
         return step_gain_dict
 
+    def identify_occlusion_pixels(self, bad_pixel_map: np.ndarray) -> np.ndarray:
+        """Find large connected bad-pixel regions caused by edge occlusion."""
+        bad_pixel_plane = np.squeeze(bad_pixel_map)
+        # Give each 8-connected region a unique label
+        labels, _ = ndimage.label(
+            bad_pixel_plane,
+            structure=np.ones((3, 3), dtype=bool),
+        )
+        # Get the label number of every region that touches an edge
+        edge_labels = np.unique(
+            np.concatenate(
+                (
+                    labels[0, :],
+                    labels[-1, :],
+                    labels[:, 0],
+                    labels[:, -1],
+                )
+            )
+        )
+        edge_labels = edge_labels[edge_labels != 0]
+
+        occlusion_plane = np.zeros_like(bad_pixel_plane, dtype=bool)
+        component_sizes = np.bincount(labels.ravel())
+        minimum_connected_pixels = self.parameters.occlusion_minimum_connected_pixels
+        for label in edge_labels:
+            component_size = component_sizes[label]
+            if component_size >= minimum_connected_pixels:
+                # Add the pixels in this component to the occlusion plane
+                occlusion_plane |= labels == label
+                logger.info(f"Identified {component_size} edge-connected occlusion pixels")
+
+        return occlusion_plane.reshape(bad_pixel_map.shape)
+
+    @staticmethod
+    def neutralize_occlusion_gain_values(
+        gain_array_dict: dict[int, np.ndarray],
+        step_occlusion_mask_dict: dict[int, np.ndarray],
+    ) -> dict[int, np.ndarray]:
+        """Set occluded pixels to a neutral gain correction."""
+        for step, occlusion_mask in step_occlusion_mask_dict.items():
+            gain_array_dict[step][occlusion_mask] = 1.0
+        return gain_array_dict
+
     @staticmethod
     def replace_invalid_gain_values(
         gain_array_dict: dict[int, np.ndarray],

{dkist_processing_vbi-1.34.8 → dkist_processing_vbi-2.0.0}/dkist_processing_vbi/tests/conftest.py

@@ -346,6 +346,7 @@ class VbiConstantsDb:
 class VbiInputDatasetParameterValues:
     vbi_bad_pixel_neighborhood_radius: int = 5
     vbi_bad_pixel_standard_deviations: float = 9.0
+    vbi_occlusion_minimum_connected_pixels: int = 1000
     vbi_movie_intensity_clipping_percentile: float = 0.5
     vbi_central_region_side_length: int = 300
 

{dkist_processing_vbi-1.34.8 → dkist_processing_vbi-2.0.0}/dkist_processing_vbi/tests/test_assemble_movie.py

@@ -3,6 +3,9 @@ import pytest
 from astropy.visualization.wcsaxes import WCSAxes
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
+from dkist_processing_common.tests.mock_metadata_store import (
+    make_default_input_dataset_recipe_run_response,
+)
 from imageio_ffmpeg import count_frames_and_secs
 from imageio_ffmpeg import read_frames
 
@@ -14,9 +17,22 @@ from dkist_processing_vbi.tests.conftest import VbiInputDatasetParameterValues
 from dkist_processing_vbi.tests.conftest import generate_214_l1_fits_frame
 
 
+@pytest.fixture(
+    scope="function",
+    params=[1, 10],
+    ids=["1_mosaic_repeat", "10_mosaic_repeats"],
+)
+def num_mosaic_repeats(request):
+    return request.param
+
+
 @pytest.fixture(scope="function")
 def assemble_task_with_tagged_movie_frames(
-    tmp_path, recipe_run_id, init_vbi_constants_db, num_mosaic_repeats
+    tmp_path,
+    recipe_run_id,
+    init_vbi_constants_db,
+    assign_input_dataset_doc_to_task,
+    num_mosaic_repeats,
 ):
     constants_db = VbiConstantsDb(NUM_MOSAIC_REPEATS=num_mosaic_repeats)
     init_vbi_constants_db(recipe_run_id, constants_db)
@@ -47,26 +63,20 @@ def assemble_task_with_tagged_movie_frames(
                 ],
                 encoder=fits_hdulist_encoder,
             )
+        assign_input_dataset_doc_to_task(task, VbiInputDatasetParameterValues())
         yield task
         task._purge()
 
 
-@pytest.mark.parametrize(
-    "num_mosaic_repeats",
-    [pytest.param(1, id="1_mosaic_repeat"), pytest.param(10, id="10_mosaic_repeats")],
-)
 def test_assemble_movie(
     assemble_task_with_tagged_movie_frames,
     mocker,
-    assign_input_dataset_doc_to_task,
     fake_gql_client,
-    num_mosaic_repeats,
 ):
     mocker.patch(
         "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
     )
     task = assemble_task_with_tagged_movie_frames
-    assign_input_dataset_doc_to_task(task, VbiInputDatasetParameterValues())
     task()
     assert task.parameters.movie_intensity_clipping_percentile == 0.5
     movie_file = list(assemble_task_with_tagged_movie_frames.read(tags=[VbiTag.movie()]))
@@ -82,3 +92,71 @@ def test_assemble_movie(
     assert f"DKIST/VBI-RED Level 1 Quicklook -- Dataset {task.constants.dataset_id}" in figure_text
     assert isinstance(task.movie_axis, WCSAxes)
     assert all(frame.data is None for frame in task.movie_frames)
+
+
+@pytest.mark.parametrize(
+    ("number_of_spatial_steps", "scanning_mode"),
+    [
+        pytest.param(1, "Center only", id="center_only"),
+        pytest.param(2, "Field Scanning", id="field_scanning"),
+    ],
+)
+def test_format_dataset_info_includes_scanning_mode(
+    assemble_task_with_tagged_movie_frames,
+    number_of_spatial_steps,
+    scanning_mode,
+):
+    """
+    Given: A set of movie frames with a specified number of spatial steps
+    When: Formatting the dataset info
+    Then: The scanning mode is correctly included in the dataset info string
+    """
+    task = assemble_task_with_tagged_movie_frames
+    task.metadata_store_input_dataset_recipe_run = make_default_input_dataset_recipe_run_response()
+    task.movie_frames = task.read_movie_frames()
+    first_frame = task.movie_frames[0]
+    first_frame.number_of_spatial_steps = number_of_spatial_steps
+
+    dataset_info = task.format_dataset_info(first_frame)
+
+    assert f"Scanning Mode: {scanning_mode}\n" in dataset_info
+
+
+@pytest.mark.parametrize(
+    "non_finite_value",
+    [
+        pytest.param(np.nan, id="nan"),
+        pytest.param(np.inf, id="positive_infinity"),
+        pytest.param(-np.inf, id="negative_infinity"),
+    ],
+)
+def test_apply_colormap_ignores_non_finite_values_when_scaling(
+    assemble_task_with_tagged_movie_frames, non_finite_value
+):
+    """
+    Given: An array with non-finite values (NaN, positive infinity, negative infinity)
+    When: Applying the colormap to the array
+    Then: The non-finite values are ignored when scaling, and they are mapped to white
+    """
+    array = np.array([[0.0, 0.5, 1.0, non_finite_value]])
+
+    result = assemble_task_with_tagged_movie_frames.apply_colormap(array)
+
+    expected = np.array([[[0, 0, 0], [128, 128, 128], [255, 255, 255], [255, 255, 255]]])
+    np.testing.assert_array_equal(result, expected)
+
+
+def test_apply_colormap_handles_array_without_finite_values(
+    assemble_task_with_tagged_movie_frames,
+):
+    """
+    Given: An array without any finite values (only NaN, positive infinity, negative infinity)
+    When: Applying the colormap to the array
+    Then: The result is an array of white pixels for all positions
+    """
+    array = np.array([[np.nan, np.inf, -np.inf]])
+
+    result = assemble_task_with_tagged_movie_frames.apply_colormap(array)
+
+    expected = np.full((1, 3, 3), 255)
+    np.testing.assert_array_equal(result, expected)

{dkist_processing_vbi-1.34.8 → dkist_processing_vbi-2.0.0}/dkist_processing_vbi/tests/test_gain.py

@@ -138,7 +138,11 @@ def test_gain_calibration(gain_calibration_task, mocker, fake_gql_client):
 
 
 def test_identify_bad_pixels(gain_calibration_task):
-    """Pixels matching any bad-pixel condition are marked in the output map."""
+    """
+    Given: a set of gain frames with known bad pixels
+    When: the identify_bad_pixels method is called
+    Then: the bad pixels are correctly identified in the output map
+    """
     averaged_gain_array = np.full((100, 100), 100.0)
     dark_subtracted_gain_array = averaged_gain_array.copy()
 
@@ -160,3 +164,48 @@ def test_identify_bad_pixels(gain_calibration_task):
     expected_bad_pixel_map = np.zeros_like(averaged_gain_array, dtype=bool)
     expected_bad_pixel_map[0, :5] = True
     np.testing.assert_array_equal(bad_pixel_map, expected_bad_pixel_map)
+
+
+@pytest.mark.parametrize(
+    "bad_pixel_map_shape",
+    [
+        pytest.param((200, 200), id="2d"),
+        pytest.param((1, 200, 200), id="leading_dimension"),
+    ],
+)
+def test_identify_occlusion_pixels(gain_calibration_task, bad_pixel_map_shape):
+    """
+    Given: a bad pixel map with known occluded regions
+    When: the identify_occlusion_pixels method is called
+    Then: the occluded regions are correctly identified in the output mask
+    """
+    bad_pixel_map = np.zeros(bad_pixel_map_shape, dtype=bool)
+    bad_pixel_map[..., 0:20, 75:125] = True  # large edge region - occlusion
+    bad_pixel_map[..., 50:75, 80:120] = True  # large interior region
+    bad_pixel_map[..., 150:160, 0:99] = True  # small edge region
+
+    occlusion_mask = gain_calibration_task.identify_occlusion_pixels(bad_pixel_map)
+
+    expected_occlusion_mask = np.zeros_like(bad_pixel_map)
+    expected_occlusion_mask[..., 0:20, 75:125] = True
+    assert occlusion_mask.shape == bad_pixel_map_shape
+    np.testing.assert_array_equal(occlusion_mask, expected_occlusion_mask)
+
+
+def test_neutralize_occlusion_gain_values():
+    """
+    Given: a gain array with known occluded regions
+    When: the neutralize_occlusion_gain_values method is called
+    Then: the occluded regions are set to 1.0 and the other values remain unchanged
+    """
+    gain_array = np.array([[[0.5, 0.75, 1.25], [1.5, 1.75, 2.0]]])
+    original_gain_array = gain_array.copy()
+    occlusion_mask = np.array([[[False, True, False], [True, False, False]]])
+
+    result = GainCalibration.neutralize_occlusion_gain_values(
+        gain_array_dict={1: gain_array},
+        step_occlusion_mask_dict={1: occlusion_mask},
+    )
+
+    np.testing.assert_array_equal(result[1][occlusion_mask], 1.0)
+    np.testing.assert_array_equal(result[1][~occlusion_mask], original_gain_array[~occlusion_mask])