dkist-processing-cryonirsp 1.1.0__tar.gz → 1.1.1__tar.gz

{dkist_processing_cryonirsp-1.1.0 → dkist_processing_cryonirsp-1.1.1}/CHANGELOG.rst

@@ -1,3 +1,14 @@
+v1.1.1 (2024-09-18)
+===================
+
+Bugfixes
+--------
+
+- Add validation in linearity_correction task to ensure that the value of NUM_FRAMES_IN_RAMP is the same
+  across all frames in a ramp, and that the value of NUM_FRAMES_IN_RAMP actually matches the number of frames found. (`#147 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/147>`__)
+- When writing linearized frames, use a fixed tag list as opposed to one derived from L0 data to alleviate load on redis. (`#148 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/148>`__)
+
+
 v1.1.0 (2024-09-10)
 ===================
 

{dkist_processing_cryonirsp-1.1.0 → dkist_processing_cryonirsp-1.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist_processing_cryonirsp
-Version: 1.1.0
+Version: 1.1.1
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA

{dkist_processing_cryonirsp-1.1.0 → dkist_processing_cryonirsp-1.1.1}/dkist_processing_cryonirsp/tasks/linearity_correction.py

@@ -25,7 +25,6 @@ class _RampSet:
     current_ramp_set_num: int
     time_obs: str
     num_frames_in_ramp: int
-    expected_num_frames_in_ramp: int
     exposure_times_ms: np.ndarray
     frame_shape: tuple[int]
     last_frame_name: str
@@ -33,10 +32,6 @@ class _RampSet:
     frames_to_process: np.ndarray
     index_offset_to_first_frame: int
 
-    @property
-    def is_valid(self):
-        return self.num_frames_in_ramp == self.expected_num_frames_in_ramp
-
 
 class LinearityCorrection(CryonirspTaskBase):
     """Task class for performing linearity correction on all input frames, regardless of task type."""
@@ -76,10 +71,12 @@ class LinearityCorrection(CryonirspTaskBase):
             exposure_corrected_output_array = self.apply_exposure_corrections(
                 output_array, ramp_set
             )
-            # Get the tags for the linearized output frame
-            tags = self.generate_output_tag_set(
-                ramp_set.last_frame_name, ramp_set.frames_to_process[-1]
-            )
+            # Set the tags for the linearized output frame
+            tags = [
+                CryonirspTag.linearized(),
+                CryonirspTag.frame(),
+                CryonirspTag.time_obs(time_obs),
+            ]
             # The last frame in the ramp is used for the header
             self.write(
                 data=exposure_corrected_output_array,
@@ -109,11 +106,53 @@ class LinearityCorrection(CryonirspTaskBase):
         Generator which yields _RampSet instances
         """
         for ramp_set_num, time_obs in enumerate(self.constants.time_obs_list):
+            input_objects = list(
+                self.read(
+                    tags=[
+                        CryonirspTag.input(),
+                        CryonirspTag.frame(),
+                        CryonirspTag.time_obs(time_obs),
+                    ],
+                    decoder=fits_access_decoder,
+                    fits_access_class=CryonirspRampFitsAccess,
+                )
+            )
+
+            if not self.is_ramp_valid(input_objects):
+                continue
+
             ramp_set = self.populate_ramp_set(time_obs, ramp_set_num)
-            if ramp_set.is_valid:
-                yield ramp_set
-            else:
-                logger.info(f"Ramp set for {time_obs=} skipped.")
+            yield ramp_set
+
+    def is_ramp_valid(self, ramp_object_list: list[CryonirspRampFitsAccess]) -> bool:
+        """
+        Check if a given ramp is valid.
+
+        Current validity checks are:
+
+          1. All frames in the ramp have the same value for NUM_FRAMES_IN_RAMP
+          2. The value of NUM_FRAMES_IN_RAMP equals the length of actual frames found
+
+        If a ramp is not valid then warnings are logged and `False` is returned.
+        """
+        frames_in_ramp_set = {o.num_frames_in_ramp for o in ramp_object_list}
+        task_type = ramp_object_list[0].ip_task_type
+
+        if len(frames_in_ramp_set) > 1:
+            logger.info(
+                f"Not all frames have the same FRAMES_IN_RAMP value. Set is {frames_in_ramp_set}. Ramp is task {task_type}. Skipping ramp."
+            )
+            return False
+
+        num_frames_in_ramp = frames_in_ramp_set.pop()
+        num_ramp_objects = len(ramp_object_list)
+        if num_ramp_objects != num_frames_in_ramp:
+            logger.info(
+                f"Missing some ramp frames. Expected {num_frames_in_ramp} from header value, but only have {num_ramp_objects}. Ramp is task {task_type}. Skipping ramp."
+            )
+            return False
+
+        return True
 
     @staticmethod
     def tag_list_for_single_ramp_frame(time_obs: str, frame_num: int) -> list[CryonirspTag]:
@@ -201,7 +240,6 @@ class LinearityCorrection(CryonirspTaskBase):
         actual_num_frames_in_ramp = self.count(CryonirspTag.time_obs(time_obs))
         exp_times = self.get_ordered_exposure_time_list(time_obs, actual_num_frames_in_ramp)
         last_frame = self.read_single_ramp_frame(time_obs, actual_num_frames_in_ramp)
-        expected_num_frames_in_ramp = last_frame.num_frames_in_ramp
         ramp_set_num = idx + 1
         # The default list of curr_frame_in_ramp tag numbers to use, which may be altered later on
         frames_to_process = np.array(range(1, actual_num_frames_in_ramp + 1), dtype=int)
@@ -209,7 +247,6 @@ class LinearityCorrection(CryonirspTaskBase):
             current_ramp_set_num=ramp_set_num,
             time_obs=time_obs,
             num_frames_in_ramp=actual_num_frames_in_ramp,
-            expected_num_frames_in_ramp=expected_num_frames_in_ramp,
             exposure_times_ms=exp_times,
             frame_shape=last_frame.data.shape,
             last_frame_name=last_frame.name,
@@ -220,25 +257,6 @@ class LinearityCorrection(CryonirspTaskBase):
         )
         return ramp_set
 
-    def generate_output_tag_set(self, frame_name: str, curr_frame_in_ramp: int) -> list[str]:
-        """
-        Use one of the input frames to derive the proper list of output tags.
-
-        Parameters
-        ----------
-        frame_name
-            file path to use to get starting tag set
-
-        Returns
-        -------
-        A list of tags to use for writing the output frame
-        """
-        result_tags = list(self.tags(frame_name))
-        result_tags.remove(CryonirspTag.input())
-        result_tags.remove(CryonirspTag.curr_frame_in_ramp(curr_frame_in_ramp))
-        result_tags.append(CryonirspTag.linearized())
-        return result_tags
-
     def apply_exposure_corrections(self, input_array: np.ndarray, ramp_set: _RampSet) -> np.ndarray:
         """
         Normalize the array by converting to counts per second and correcting for Optical Density filter attenuation.

{dkist_processing_cryonirsp-1.1.0 → dkist_processing_cryonirsp-1.1.1}/dkist_processing_cryonirsp/tests/test_linearity_correction.py

@@ -1,4 +1,6 @@
 """Test the linearity correction task."""
+import re
+from dataclasses import dataclass
 from datetime import datetime
 
 import numpy as np
@@ -9,6 +11,7 @@ from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tests.conftest import FakeGQLClient
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.models.constants import CryonirspBudName
 from dkist_processing_cryonirsp.models.exposure_conditions import AllowableOpticalDensityFilterNames
@@ -138,7 +141,12 @@ def linearity_correction(
     [pytest.param(1, id="1 chunk"), pytest.param(2, id="2 chunks")],
 )
 def test_linearity_correction(
-    linearity_correction, mocker, arm_id, frames_in_ramp, num_chunks, filter_name
+    linearity_correction,
+    mocker,
+    arm_id,
+    frames_in_ramp,
+    num_chunks,
+    filter_name,
 ):
     """
     Given: A LinearityCorrection task
@@ -176,3 +184,62 @@ def test_linearity_correction(
         assert np.allclose(data, expected_data)
     else:
         assert len(files_found) == 0
+
+
+@pytest.fixture
+def simple_linearity_correction_task(recipe_run_id, arm_id, init_cryonirsp_constants_db, tmp_path):
+    constants_db = CryonirspConstantsDb(
+        ARM_ID=arm_id,
+    )
+    init_cryonirsp_constants_db(recipe_run_id, constants_db)
+    with LinearityCorrection(
+        recipe_run_id=recipe_run_id,
+        workflow_name="workflow_name",
+        workflow_version="workflow_version",
+    ) as task:
+        task.scratch = WorkflowFileSystem(recipe_run_id=recipe_run_id, scratch_base_path=tmp_path)
+
+        yield task
+
+        task._purge()
+
+
+@dataclass
+class DummyRampFitsAccess:
+    """Just a class that has the one property that is checked during ramp validation."""
+
+    num_frames_in_ramp: int
+    ip_task_type: str = "TASK"
+
+
+@pytest.mark.parametrize(
+    "arm_id",
+    [pytest.param("CI", id="CI"), pytest.param("SP", id="SP")],
+)
+@pytest.mark.parametrize(
+    "ramp_list, valid, message",
+    [
+        pytest.param(
+            [
+                DummyRampFitsAccess(num_frames_in_ramp=2),
+                DummyRampFitsAccess(num_frames_in_ramp=3),
+            ],
+            False,
+            "Not all frames have the same FRAMES_IN_RAMP value. Set is {2, 3}. Ramp is task TASK. Skipping ramp.",
+            id="num_frames_mismatch_actual_frames",
+        ),
+        pytest.param(
+            [
+                DummyRampFitsAccess(num_frames_in_ramp=8),
+            ],
+            False,
+            "Missing some ramp frames. Expected 8 from header value, but only have 1. Ramp is task TASK. Skipping ramp.",
+            id="num_frames_in_set_mismatch",
+        ),
+    ],
+)
+def test_is_ramp_valid(simple_linearity_correction_task, ramp_list, valid, message, caplog):
+    logger.add(caplog.handler)
+    assert simple_linearity_correction_task.is_ramp_valid(ramp_list) is valid
+    if not valid:
+        assert re.search(message, caplog.text)

{dkist_processing_cryonirsp-1.1.0 → dkist_processing_cryonirsp-1.1.1}/dkist_processing_cryonirsp.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-cryonirsp
-Version: 1.1.0
+Version: 1.1.1
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA