dkist-processing-cryonirsp 0.0.4__tar.gz → 0.0.5__tar.gz

{dkist_processing_cryonirsp-0.0.4 → dkist_processing_cryonirsp-0.0.5}/CHANGELOG.rst

@@ -1,3 +1,12 @@
+v0.0.5 (2023-09-06)
+===================
+
+Misc
+----
+
+- Refactor linearity correction to improve memory usage. (`#59 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/59>`__)
+
+
 v0.0.4 (2023-09-06)
 ===================
 

{dkist_processing_cryonirsp-0.0.4 → dkist_processing_cryonirsp-0.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist_processing_cryonirsp
-Version: 0.0.4
+Version: 0.0.5
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA

{dkist_processing_cryonirsp-0.0.4 → dkist_processing_cryonirsp-0.0.5}/dkist_processing_cryonirsp/models/parameters.py

@@ -159,13 +159,13 @@ class CryonirspParameters(ParameterBase):
     def linearization_thresholds(self) -> np.ndarray:
         """Name of parameter associated with the linearization thresholds."""
         param_dict = self._find_parameter_for_arm("cryonirsp_linearization_thresholds")
-        return self._load_param_value_from_npy_file(param_dict)
+        return self._load_param_value_from_npy_file(param_dict).astype(np.float32)
 
     @cached_property
     def linearization_polyfit_coeffs(self) -> np.ndarray:
         """Name of parameter associated with the linearization polyfit coefficients."""
         param_dict = self._find_parameter_for_arm("cryonirsp_linearization_polyfit_coeffs")
-        return self._load_param_value_from_npy_file(param_dict)
+        return self._load_param_value_from_npy_file(param_dict).astype(np.float32)
 
     def _find_parameter_closest_wavelength(self, parameter_name: str) -> Any:
         """

{dkist_processing_cryonirsp-0.0.4 → dkist_processing_cryonirsp-0.0.5}/dkist_processing_cryonirsp/tasks/linearity_correction.py

@@ -1,4 +1,5 @@
 """CryoNIRSP Linearity COrrection Task."""
+import sys
 from dataclasses import dataclass
 from typing import Generator
 
@@ -52,6 +53,7 @@ class LinearityCorrection(WorkflowTaskBase, InputFrameMixin, InputDatasetMixin,
         self.parameters = CryonirspParameters(
             self.input_dataset_parameters, arm_id=self.constants.arm_id
         )
+        self.current_ramp_number = None
 
     def run(self):
         """Run method for this task.
@@ -71,6 +73,7 @@ class LinearityCorrection(WorkflowTaskBase, InputFrameMixin, InputDatasetMixin,
             with self.apm_task_step(
                 f"Processing frames from {ramp_set.time_obs}: ramp set {ramp_set.current_ramp_set_num} of {ramp_set.total_ramp_sets}"
             ):
+                self.current_ramp_number = ramp_set.current_ramp_set_num
                 with self.apm_task_step(f"Gathering Inputs"):
                     ramp_objs = self._gather_inputs_for_ramp_set(ramp_set=ramp_set)
                 with self.apm_processing_step("Performing Linearization Correction"):
@@ -210,11 +213,13 @@ class LinearityCorrection(WorkflowTaskBase, InputFrameMixin, InputDatasetMixin,
         """Process a single ramp from a set of input frames whose mode is 'LookUpTable' and camera readout mode is 'FastUpTheRamp'."""
         # drop first frame in FastUpTheRamp
         ramp_objs = ramp_objs[1:]
-
-        exptimes = np.array([obj.fpa_exposure_time_ms for obj in ramp_objs])
+        exptimes = np.array([obj.fpa_exposure_time_ms for obj in ramp_objs]).astype(np.float32)
 
         # Create a 3D stack of the ramp frames that has shape (x, y, num_ramps)
-        raw_data = np.zeros((ramp_objs[0].data.shape + (len(ramp_objs),)))
+        raw_data = np.zeros((ramp_objs[0].data.shape + (len(ramp_objs),)), dtype=np.float32)
+        logger.info(
+            f"Size of ramp data cube for ramp {self.current_ramp_number} is {sys.getsizeof(raw_data)} bytes"
+        )
         for n, obj in enumerate(ramp_objs):
             raw_data[:, :, n] = obj.data
 
@@ -230,14 +235,13 @@ class LinearityCorrection(WorkflowTaskBase, InputFrameMixin, InputDatasetMixin,
             roi_1_origin_y : (roi_1_origin_y + roi_1_size_y),
             roi_1_origin_x : (roi_1_origin_x + roi_1_size_x),
         ]
-
         # correct the whole data first using the curve derived from the on Sun data
         linc = np.flip(lin_curve)
-        raw_data = self._lin_correct(raw_data, linc)
-
+        raw_data = self._lin_correct_by_row(raw_data, linc)
         slopes = self._get_slopes(exptimes, raw_data, thres_roi)
         # Scale the slopes by the exposure time to convert to counts
         processed_frame = slopes * np.nanmax(exptimes)
+
         return processed_frame
 
     def _reduce_ramp_set(
@@ -282,3 +286,15 @@ class LinearityCorrection(WorkflowTaskBase, InputFrameMixin, InputDatasetMixin,
         raise ValueError(
             f"Linearization mode {mode} and camera readout mode {camera_readout_mode} is currently not supported."
         )
+
+    def _lin_correct_by_row(self, raw_data: np.ndarray, linc: np.ndarray) -> np.ndarray:
+        """
+        Perform linearity correction on the data cube one row at a time.
+
+        The cube of cryonirsp ramp data can be very large (2k x 2k x 100+ elements) and so to
+        better constrain memory only one row-based-slice of the cube is corrected at any given time.
+        :return:
+        """
+        for j in range(raw_data.shape[0]):
+            raw_data[j, :] = self._lin_correct(raw_data[j, :], linc)
+        return raw_data

{dkist_processing_cryonirsp-0.0.4 → dkist_processing_cryonirsp-0.0.5}/dkist_processing_cryonirsp.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-cryonirsp
-Version: 0.0.4
+Version: 0.0.5
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA