dkist-processing-cryonirsp 0.0.51__tar.gz → 0.0.52__tar.gz

{dkist_processing_cryonirsp-0.0.51 → dkist_processing_cryonirsp-0.0.52}/CHANGELOG.rst

@@ -1,3 +1,12 @@
+v0.0.52 (2024-04-02)
+====================
+
+Misc
+----
+
+- Update bad pixel correction method such that if more than a given fraction of the frame is impacted, a faster and more general algorithm is used. This reduces processing time in cases where the data has readout problems or other large scale issues. (`#109 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/109>`__)
+
+
 v0.0.51 (2024-03-27)
 ====================
 

{dkist_processing_cryonirsp-0.0.51 → dkist_processing_cryonirsp-0.0.52}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist_processing_cryonirsp
-Version: 0.0.51
+Version: 0.0.52
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA

{dkist_processing_cryonirsp-0.0.51 → dkist_processing_cryonirsp-0.0.52}/dkist_processing_cryonirsp/models/parameters.py

@@ -127,6 +127,17 @@ class CryonirspParameters(ParameterBase):
             "cryonirsp_corrections_bad_pixel_median_filter_size"
         )
 
+    @cached_property
+    def corrections_bad_pixel_fraction_threshold(self) -> float:
+        """
+        Return the fraction of the bad pixel mask.
+
+        If exceeded, will cause the fallback to a faster but worse algorithm.
+        """
+        return self._find_most_recent_past_value(
+            "cryonirsp_corrections_bad_pixel_fraction_threshold"
+        )
+
     @cached_property
     def linearization_thresholds(self) -> np.ndarray:
         """Name of parameter associated with the linearization thresholds."""

{dkist_processing_cryonirsp-0.0.51 → dkist_processing_cryonirsp-0.0.52}/dkist_processing_cryonirsp/tasks/mixin/corrections.py

@@ -129,7 +129,10 @@ class CorrectionsMixin:
         """
         Correct bad pixels in an array using a median filter.
 
-        Corrects only the bad pixels identified by the bad pixel map
+        Corrects only the bad pixels identified by the bad pixel map.
+
+        If enough pixels are found to suggest readout errors (> a given fraction of the array),
+        a global replacement value is used instead.
 
         Parameters
         ----------
@@ -145,6 +148,12 @@ class CorrectionsMixin:
         ndarray
             The corrected array
         """
+        bad_pixel_fraction = np.sum(bad_pixel_map) / bad_pixel_map.size
+        if bad_pixel_fraction > self.parameters.corrections_bad_pixel_fraction_threshold:
+            global_median = np.nanmedian(array_to_fix)
+            array_to_fix[bad_pixel_map == 1] = global_median
+            return array_to_fix
+
         kernel_size = self.parameters.corrections_bad_pixel_median_filter_size
         half_kernel_size = kernel_size // 2
 

{dkist_processing_cryonirsp-0.0.51 → dkist_processing_cryonirsp-0.0.52}/dkist_processing_cryonirsp/tests/conftest.py

@@ -294,7 +294,7 @@ def cryonirsp_testing_parameters_factory(
         cryonirsp_geo_strip_short_axis_size_fraction: float = 0.1
         cryonirsp_geo_strip_spectral_offset_size_fraction: float = 0.25
         cryonirsp_solar_characteristic_spatial_normalization_percentile: float = 80.0
-        cryonirsp_max_cs_step_time_sec: float = 160.0
+        cryonirsp_max_cs_step_time_sec: float = 180.0
         cryonirsp_beam_boundaries_smoothing_disk_size: int = 3
         cryonirsp_beam_boundaries_upsample_factor: int = 10
         cryonirsp_beam_boundaries_sp_beam_transition_region_size_fraction: float = 0.05
@@ -306,6 +306,7 @@ def cryonirsp_testing_parameters_factory(
         )
         cryonirsp_bad_pixel_map_threshold_factor: float = 5.0
         cryonirsp_corrections_bad_pixel_median_filter_size: int = 8
+        cryonirsp_corrections_bad_pixel_fraction_threshold: float = 0.03
         cryonirsp_fringe_correction_on: bool = True
         cryonirsp_fringe_correction_spectral_filter_size: list[int] = field(
             default_factory=lambda: [1, 20]

dkist_processing_cryonirsp-0.0.52/dkist_processing_cryonirsp/tests/test_corrections.py

@@ -0,0 +1,92 @@
+import numpy as np
+import pytest
+from dkist_processing_common._util.scratch import WorkflowFileSystem
+from dkist_processing_common.tasks import WorkflowTaskBase
+from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetMixin
+from dkist_processing_common.tests.conftest import FakeGQLClient
+
+from dkist_processing_cryonirsp.models.constants import CryonirspConstants
+from dkist_processing_cryonirsp.tasks.mixin.corrections import CorrectionsMixin
+from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
+from dkist_processing_cryonirsp.tests.conftest import CryonirspConstantsDb
+
+base_bad_pixel_map = np.zeros(shape=(10, 10))
+
+normal_bad_pixel_map = base_bad_pixel_map.copy()
+normal_bad_pixel_map[1, 6] = 1
+
+column_error_bad_pixel_map = base_bad_pixel_map.copy()
+column_error_bad_pixel_map[:, 6] = 1
+
+
+class BadPixelMapTask(WorkflowTaskBase, CorrectionsMixin, InputDatasetMixin):
+    constants: CryonirspConstants
+
+    @property
+    def constants_model_class(self):
+        """Get CryoNIRSP pipeline constants."""
+        return CryonirspConstants
+
+    def run(self):
+        pass
+
+
+@pytest.fixture(params=["CI", "SP"])
+def bad_pixel_mask_task(
+    tmp_path,
+    recipe_run_id,
+    assign_input_dataset_doc_to_task,
+    mocker,
+    request,
+    init_cryonirsp_constants_db,
+):
+    mocker.patch(
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
+    )
+    constants_db = CryonirspConstantsDb(
+        ARM_ID=request.param,
+    )
+    init_cryonirsp_constants_db(recipe_run_id, constants_db)
+    with BadPixelMapTask(
+        recipe_run_id=recipe_run_id,
+        workflow_name="bad_pixel_mask",
+        workflow_version="VX.Y",
+    ) as task:
+        try:  # This try... block is here to make sure the dbs get cleaned up if there's a failure in the fixture
+            task.scratch = WorkflowFileSystem(
+                scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
+            )
+            param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
+            assign_input_dataset_doc_to_task(task, param_class())
+            yield task
+        finally:
+            task._purge()
+
+
+@pytest.mark.parametrize(
+    "bad_pixel_map, algorithm_type",
+    [
+        pytest.param(normal_bad_pixel_map, "normal", id="normal algorithm"),
+        pytest.param(column_error_bad_pixel_map, "fast", id="fast algorithm"),
+    ],
+)
+def test_corrections_correct_bad_pixels(bad_pixel_map, algorithm_type, bad_pixel_mask_task):
+    t = bad_pixel_mask_task
+    bad_pixel_x = 1
+    bad_pixel_y = 6
+    array_to_fix = np.random.random(size=(10, 10))
+    # Assign a single bad pixel to check against
+    array_to_fix[bad_pixel_x, bad_pixel_y] = 0
+    corrected_array = t.corrections_correct_bad_pixels(
+        array_to_fix=array_to_fix, bad_pixel_map=bad_pixel_map
+    )
+    if algorithm_type == "fast":
+        for val in corrected_array[:, bad_pixel_y]:
+            assert val == np.nanmedian(array_to_fix)
+        assert corrected_array[bad_pixel_x, bad_pixel_y] != 0
+    if algorithm_type == "normal":
+        x, y = np.meshgrid(np.arange(10), np.arange(10))
+        idx = np.where((x != bad_pixel_x) | (y != bad_pixel_y))
+        np.testing.assert_array_equal(corrected_array[idx], array_to_fix[idx])
+        assert corrected_array[bad_pixel_x, bad_pixel_y] != np.nanmedian(array_to_fix)
+        assert corrected_array[bad_pixel_x, bad_pixel_y] != 0

{dkist_processing_cryonirsp-0.0.51 → dkist_processing_cryonirsp-0.0.52}/dkist_processing_cryonirsp.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-cryonirsp
-Version: 0.0.51
+Version: 0.0.52
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA

{dkist_processing_cryonirsp-0.0.51 → dkist_processing_cryonirsp-0.0.52}/dkist_processing_cryonirsp.egg-info/SOURCES.txt

@@ -72,6 +72,7 @@ dkist_processing_cryonirsp/tests/test_bad_pixel_maps.py
 dkist_processing_cryonirsp/tests/test_build_quality_report.py
 dkist_processing_cryonirsp/tests/test_ci_beam_boundaries.py
 dkist_processing_cryonirsp/tests/test_ci_science.py
+dkist_processing_cryonirsp/tests/test_corrections.py
 dkist_processing_cryonirsp/tests/test_cryo_base.py
 dkist_processing_cryonirsp/tests/test_cryo_constants.py
 dkist_processing_cryonirsp/tests/test_dark.py