dkist-processing-cryonirsp 0.0.37rc1__tar.gz → 0.0.38__tar.gz

{dkist_processing_cryonirsp-0.0.37rc1 → dkist_processing_cryonirsp-0.0.38}/CHANGELOG.rst

@@ -1,3 +1,23 @@
+v0.0.38 (2024-02-22)
+====================
+
+Bugfixes
+--------
+
+- Fixed bugs in `SPGeometricCalibration` task to use basic-corrected or gain-corrected arrays where needed. (`#97 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/97>`__)
+- Fixed errors in `CorrectionsMixin` that caused problems in rotation, shift and spectral curvature computations. (`#97 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/97>`__)
+
+
+v0.0.37 (2024-02-21)
+====================
+
+Bugfixes
+--------
+
+- Added ObsIpStartTimeBud to ramp parser to support parameter access outside of parse tasks (`#95 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/95>`__)
+- Fix local workflow code to use the correct polyfit coefficients for local data processing. (`#96 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/96>`__)
+
+
 v0.0.36 (2024-02-20)
 ====================
 

{dkist_processing_cryonirsp-0.0.37rc1 → dkist_processing_cryonirsp-0.0.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist_processing_cryonirsp
-Version: 0.0.37rc1
+Version: 0.0.38
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA

{dkist_processing_cryonirsp-0.0.37rc1 → dkist_processing_cryonirsp-0.0.38}/dkist_processing_cryonirsp/tasks/mixin/corrections.py

@@ -27,11 +27,11 @@ class CorrectionsMixin:
         arrays
             2D array(s) containing the data for the un-shifted beam
 
-        shift : np.ndarray
+        shift
             The measured shift offset in the spectral dimension
             between beams or between modulator states in a single beam.
 
-        angle : float
+        angle
             The angle between the slit and pixel axes.
 
         Returns
@@ -46,18 +46,9 @@ class CorrectionsMixin:
             array[np.where(array == np.inf)] = np.max(array[np.isfinite(array)])
             array[np.where(array == -np.inf)] = np.min(array[np.isfinite(array)])
             array[np.isnan(array)] = np.nanmedian(array)
-            cval = np.nanmedian(array)
-            translated = affine_transform_arrays(
-                # The shift is inverted here to represent the correction that should be applied
-                array,
-                translation=-shift,
-                mode="constant",
-                cval=cval,
-            )
-            yield next(
-                # TODO: rotate_arrays_about_point rotates the wrong way, so no negative sign here
-                rotate_arrays_about_point(translated, angle=angle, mode="constant", cval=cval)
-            )
+            translated = affine_transform_arrays(array, translation=-shift, mode="edge", order=1)
+            # TODO: rotate_arrays_about_point rotates the wrong way, so no negative sign here
+            yield next(rotate_arrays_about_point(translated, angle=angle, mode="edge", order=1))
 
     @staticmethod
     def corrections_distort_geometry(
@@ -75,11 +66,11 @@ class CorrectionsMixin:
         arrays
             2D array(s) containing the data for the un-shifted beam
 
-        shift : np.ndarray
+        shift
             The measured shift offset in the spectral dimension
             between beams or between modulator states in a single beam.
 
-        angle : float
+        angle
             The angle between the slit and pixel axes.
 
         Returns
@@ -94,18 +85,9 @@ class CorrectionsMixin:
             array[np.where(array == np.inf)] = np.max(array[np.isfinite(array)])
             array[np.where(array == -np.inf)] = np.min(array[np.isfinite(array)])
             array[np.isnan(array)] = np.nanmedian(array)
-            cval = np.nanmedian(array)
             # TODO: rotate_arrays_about_point rotates the wrong way, so no negative sign here
-            rotated = rotate_arrays_about_point(array, angle=angle, mode="constant", cval=cval)
-            yield next(
-                affine_transform_arrays(
-                    # The shift is inverted here to represent the correction that should be applied
-                    rotated,
-                    translation=-shift,
-                    mode="constant",
-                    cval=cval,
-                )
-            )
+            rotated = rotate_arrays_about_point(array, angle=angle, mode="edge")
+            yield next(affine_transform_arrays(rotated, translation=-shift, mode="edge"))
 
     @staticmethod
     def corrections_remove_spec_shifts(
@@ -136,9 +118,7 @@ class CorrectionsMixin:
             numx = array.shape[0]
             array_output = np.zeros(array.shape)
             for j in range(numx):
-                array_output[j, :] = spnd.interpolation.shift(
-                    array[j, :], -spec_shift[j], mode="constant", cval=np.nanmedian(array[j, :])
-                )
+                array_output[j, :] = spnd.shift(array[j, :], -spec_shift[j], mode="nearest")
             yield array_output
 
     def corrections_correct_bad_pixels(

{dkist_processing_cryonirsp-0.0.37rc1 → dkist_processing_cryonirsp-0.0.38}/dkist_processing_cryonirsp/tasks/sp_geometric.py

@@ -4,7 +4,6 @@ import math
 import numpy as np
 import peakutils as pku
 import scipy.ndimage as spnd
-import skimage.registration as skir
 from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_math.arithmetic import divide_arrays_by_array
 from dkist_processing_math.arithmetic import subtract_array_from_arrays
@@ -110,9 +109,9 @@ class SPGeometricCalibration(CryonirspTaskBase, ShiftMeasurementsMixin):
                 task_type=TaskName.geometric.value, total_frames=no_of_raw_geo_frames
             )
 
-    def basic_corrected_data(self, beam: int) -> np.ndarray:
+    def basic_gain_corrected_data(self, beam: int) -> np.ndarray:
         """
-        Get dark corrected data array for a single beam.
+        Get dark and lamp gain corrected data array for a single beam.
 
         Parameters
         ----------
@@ -125,7 +124,26 @@ class SPGeometricCalibration(CryonirspTaskBase, ShiftMeasurementsMixin):
             Dark corrected data array
         """
         array_generator = self.intermediate_frame_load_intermediate_arrays(
-            tags=[CryonirspTag.task("GC_BASIC"), CryonirspTag.beam(beam)]
+            tags=[CryonirspTag.task("GC_BASIC_GAIN_CORRECTED"), CryonirspTag.beam(beam)]
+        )
+        return average_numpy_arrays(array_generator)
+
+    def basic_dark_bp_corrected_data(self, beam: int) -> np.ndarray:
+        """
+        Get dark and bad pixel corrected data array for a single beam.
+
+        Parameters
+        ----------
+        beam : int
+            The current beam being processed
+
+        Returns
+        -------
+        np.ndarray
+            Dark and bad pixel corrected data array
+        """
+        array_generator = self.intermediate_frame_load_intermediate_arrays(
+            tags=[CryonirspTag.task("GC_BASIC_DARK_BP_CORRECTED"), CryonirspTag.beam(beam)]
         )
         return average_numpy_arrays(array_generator)
 
@@ -183,7 +201,12 @@ class SPGeometricCalibration(CryonirspTaskBase, ShiftMeasurementsMixin):
                 bad_pixel_corrected_array = self.corrections_correct_bad_pixels(
                     dark_corrected_solar_array, bad_pixel_map
                 )
-
+                logger.info(f"Writing bad pixel corrected data for {beam=}")
+                self.intermediate_frame_write_arrays(
+                    arrays=bad_pixel_corrected_array,
+                    beam=beam,
+                    task="GC_BASIC_DARK_BP_CORRECTED",
+                )
                 gain_corrected_solar_array = next(
                     divide_arrays_by_array(bad_pixel_corrected_array, lamp_gain_array)
                 )
@@ -191,7 +214,7 @@ class SPGeometricCalibration(CryonirspTaskBase, ShiftMeasurementsMixin):
                 self.intermediate_frame_write_arrays(
                     arrays=gain_corrected_solar_array,
                     beam=beam,
-                    task="GC_BASIC",
+                    task="GC_BASIC_GAIN_CORRECTED",
                 )
 
     def compute_beam_angle(self, beam: int) -> float:
@@ -214,7 +237,10 @@ class SPGeometricCalibration(CryonirspTaskBase, ShiftMeasurementsMixin):
         The beam rotation angle in radians
         """
         # Step 1
-        gain_array = self.basic_corrected_data(beam)
+        # Do not use a gain corrected image here, as it will cancel out the slit structure
+        # that is used for the shift measurement computations
+        gain_array = self.intermediate_frame_load_lamp_gain_array(beam=beam)
+
         full_spatial_size, full_spectral_size = gain_array.shape
 
         # Get the params for the strips
@@ -257,7 +283,7 @@ class SPGeometricCalibration(CryonirspTaskBase, ShiftMeasurementsMixin):
         np.ndarray
             The corrected array
         """
-        rotated_array = self.basic_corrected_data(beam=beam)
+        rotated_array = self.basic_gain_corrected_data(beam=beam)
         corrected_array = next(self.corrections_correct_geometry(rotated_array, angle=angle))
         return corrected_array
 

{dkist_processing_cryonirsp-0.0.37rc1 → dkist_processing_cryonirsp-0.0.38}/dkist_processing_cryonirsp/tests/local_trial_workflows/l0_cals_only.py

@@ -2,10 +2,12 @@ import argparse
 import json
 import os
 import sys
+from datetime import datetime
 from pathlib import Path
 
 from astropy.io import fits
 from dkist_processing_common.manual import ManualProcessing
+from dkist_processing_common.models.constants import BudName
 from dkist_processing_common.tasks import WorkflowTaskBase
 from dkist_service_configuration import logger
 
@@ -131,15 +133,16 @@ def tag_linearized_inputs_task(suffix: str):
     return TagLinearizedInputs
 
 
-def spoof_obs_wavelength_task(wavelength: float = 1083.0):
+def spoof_obs_ramp_parsed_constants(wavelength: float = 1083.0):
     class SetObsWavelength(WorkflowTaskBase):
         def run(self) -> None:
             self.constants._update({CryonirspBudName.wavelength.value: wavelength})
+            self.constants._update({BudName.obs_ip_start_time.value: datetime.now().isoformat()})
 
     return SetObsWavelength
 
 
-def spoof_obs_constants_task(
+def spoof_obs_lin_parsed_constants(
     polarimetric: bool = True,
 ):
     class SetCalOnlyConstants(CryonirspTaskBase):
@@ -346,7 +349,7 @@ def main(
             manual_processing_run.run_task(task=load_parsing_task(save_file="input_parsing.asdf"))
         else:
             manual_processing_run.run_task(task=ParseL0CryonirspRampData)
-            manual_processing_run.run_task(task=spoof_obs_wavelength_task(dummy_wavelength))
+            manual_processing_run.run_task(task=spoof_obs_ramp_parsed_constants(dummy_wavelength))
             manual_processing_run.run_task(
                 task=save_parsing_task(
                     tag_list=[CryonirspTag.input(), CryonirspTag.frame()],
@@ -372,7 +375,9 @@ def main(
             )
         else:
             manual_processing_run.run_task(task=ParseL0CryonirspLinearizedData)
-            manual_processing_run.run_task(task=spoof_obs_constants_task(polarimetric=polarimetric))
+            manual_processing_run.run_task(
+                task=spoof_obs_lin_parsed_constants(polarimetric=polarimetric)
+            )
             if not skip_saving_parse:
                 manual_processing_run.run_task(
                     task=save_parsing_task(

{dkist_processing_cryonirsp-0.0.37rc1 → dkist_processing_cryonirsp-0.0.38}/dkist_processing_cryonirsp/tests/local_trial_workflows/local_trial_helpers.py

@@ -18,6 +18,22 @@ from dkist_processing_cryonirsp.tasks.cryonirsp_base import CryonirspTaskBase
 from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
 
 
+# These are the workflow versions of the polyfit coefficient parameters
+WORKFLOW_LINEARIZATION_POLYFIT_COEFFS_CI = [
+    1.1565851e00,
+    -1.1925439e-05,
+    2.6748548e-10,
+    -2.7366326e-15,
+]
+
+WORKFLOW_LINEARIZATION_POLYFIT_COEFFS_SP = [
+    1.1421334e00,
+    -1.0536260e-05,
+    2.2702712e-10,
+    -2.5632187e-15,
+]
+
+
 def save_parsing_task(
     tag_list: [str], save_file: str, save_file_tags: bool = True, save_constants: bool = True
 ):
@@ -371,7 +387,13 @@ def create_input_dataset_parameter_document(param_path: Path):
             param_class = cryonirsp_testing_parameters_factory(
                 param_path=param_path, create_files=False
             )
-            for pn, pv in asdict(param_class()).items():
+            params = asdict(
+                param_class(
+                    cryonirsp_linearization_polyfit_coeffs_ci=WORKFLOW_LINEARIZATION_POLYFIT_COEFFS_CI,
+                    cryonirsp_linearization_polyfit_coeffs_sp=WORKFLOW_LINEARIZATION_POLYFIT_COEFFS_SP,
+                )
+            )
+            for pn, pv in params.items():
                 values = [
                     {
                         "parameterValueId": value_id,

{dkist_processing_cryonirsp-0.0.37rc1 → dkist_processing_cryonirsp-0.0.38}/dkist_processing_cryonirsp/tests/test_sp_geometric.py

@@ -291,7 +291,7 @@ def test_basic_corrections(geometric_calibration_task_with_simple_raw_data):
             expected = flipped_true_solar[:, 0 : beam_shape[1]]
         else:
             expected = flipped_true_solar[:, beam_shape[1] :]
-        array = task.basic_corrected_data(beam=beam)
+        array = task.basic_dark_bp_corrected_data(beam=beam)
         np.testing.assert_equal(expected, array)
 
     # Negative test on the original true solar image
@@ -300,6 +300,6 @@ def test_basic_corrections(geometric_calibration_task_with_simple_raw_data):
             expected = true_solar[:, 0 : beam_shape[1]]
         else:
             expected = true_solar[:, beam_shape[1] :]
-        array = task.basic_corrected_data(beam=beam)
+        array = task.basic_dark_bp_corrected_data(beam=beam)
         with pytest.raises(AssertionError):
             np.testing.assert_equal(expected, array)

{dkist_processing_cryonirsp-0.0.37rc1 → dkist_processing_cryonirsp-0.0.38}/dkist_processing_cryonirsp.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-cryonirsp
-Version: 0.0.37rc1
+Version: 0.0.38
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA

{dkist_processing_cryonirsp-0.0.37rc1 → dkist_processing_cryonirsp-0.0.38}/dkist_processing_cryonirsp.egg-info/SOURCES.txt

@@ -12,7 +12,6 @@ setup.cfg
 setup.py
 towncrier_science.toml
 changelog/.gitempty
-changelog/95.bugfix.rst
 dkist_processing_cryonirsp/__init__.py
 dkist_processing_cryonirsp.egg-info/PKG-INFO
 dkist_processing_cryonirsp.egg-info/SOURCES.txt

dkist_processing_cryonirsp-0.0.37rc1/changelog/95.bugfix.rst

@@ -1 +0,0 @@
-Added ObsIpStartTimeBud to ramp parser to support parameter access outside of parse tasks