dkist-processing-cryonirsp 1.19.19__tar.gz → 2.0.0rc2__tar.gz

{dkist_processing_cryonirsp-1.19.19 → dkist_processing_cryonirsp-2.0.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-cryonirsp
-Version: 1.19.19
+Version: 2.0.0rc2
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -29,6 +29,7 @@ Requires-Dist: numba==0.61.2
 Requires-Dist: numpy==2.2.5
 Requires-Dist: peakutils==1.3.5
 Requires-Dist: scikit-image==0.25.2
+Requires-Dist: scikit-learn==1.9.0
 Requires-Dist: scipy==1.15.3
 Requires-Dist: sunpy==6.1.1
 Provides-Extra: test
@@ -80,22 +81,21 @@ Requires-Dist: aiohttp==3.14.1; extra == "frozen"
 Requires-Dist: aiosignal==1.4.0; extra == "frozen"
 Requires-Dist: aiosmtplib==5.1.2; extra == "frozen"
 Requires-Dist: aiosqlite==0.21.0; extra == "frozen"
-Requires-Dist: alembic==1.18.4; extra == "frozen"
+Requires-Dist: alembic==1.18.5; extra == "frozen"
 Requires-Dist: amqp==5.3.1; extra == "frozen"
 Requires-Dist: annotated-doc==0.0.4; extra == "frozen"
 Requires-Dist: annotated-types==0.7.0; extra == "frozen"
-Requires-Dist: anyio==4.14.0; extra == "frozen"
+Requires-Dist: anyio==4.14.1; extra == "frozen"
 Requires-Dist: apache-airflow==3.2.1; extra == "frozen"
 Requires-Dist: apache-airflow-core==3.2.1; extra == "frozen"
 Requires-Dist: apache-airflow-providers-celery==3.20.0; extra == "frozen"
 Requires-Dist: apache-airflow-providers-common-compat==1.15.0; extra == "frozen"
 Requires-Dist: apache-airflow-providers-common-io==1.7.3; extra == "frozen"
 Requires-Dist: apache-airflow-providers-common-sql==2.0.1; extra == "frozen"
-Requires-Dist: apache-airflow-providers-postgres==6.7.1; extra == "frozen"
+Requires-Dist: apache-airflow-providers-postgres==6.8.0; extra == "frozen"
 Requires-Dist: apache-airflow-providers-smtp==3.0.1; extra == "frozen"
-Requires-Dist: apache-airflow-providers-standard==1.14.0; extra == "frozen"
+Requires-Dist: apache-airflow-providers-standard==1.15.0; extra == "frozen"
 Requires-Dist: apache-airflow-task-sdk==1.2.1; extra == "frozen"
-Requires-Dist: appnope==0.1.4; extra == "frozen"
 Requires-Dist: argcomplete==3.6.3; extra == "frozen"
 Requires-Dist: argon2-cffi==25.1.0; extra == "frozen"
 Requires-Dist: argon2-cffi-bindings==25.1.0; extra == "frozen"
@@ -114,14 +114,14 @@ Requires-Dist: babel==2.18.0; extra == "frozen"
 Requires-Dist: beautifulsoup4==4.15.0; extra == "frozen"
 Requires-Dist: billiard==4.2.4; extra == "frozen"
 Requires-Dist: bleach==6.4.0; extra == "frozen"
-Requires-Dist: boto3==1.43.34; extra == "frozen"
-Requires-Dist: botocore==1.43.34; extra == "frozen"
+Requires-Dist: boto3==1.43.36; extra == "frozen"
+Requires-Dist: botocore==1.43.36; extra == "frozen"
 Requires-Dist: cadwyn==7.1.0; extra == "frozen"
 Requires-Dist: celery==5.6.3; extra == "frozen"
 Requires-Dist: certifi==2026.6.17; extra == "frozen"
 Requires-Dist: cffi==2.0.0; extra == "frozen"
 Requires-Dist: charset-normalizer==3.4.7; extra == "frozen"
-Requires-Dist: click==8.4.1; extra == "frozen"
+Requires-Dist: click==8.4.2; extra == "frozen"
 Requires-Dist: click-didyoumean==0.3.1; extra == "frozen"
 Requires-Dist: click-plugins==1.1.1.2; extra == "frozen"
 Requires-Dist: click-repl==0.3.0; extra == "frozen"
@@ -139,7 +139,7 @@ Requires-Dist: dill==0.4.1; extra == "frozen"
 Requires-Dist: dkist-header-validator==5.4.0; extra == "frozen"
 Requires-Dist: dkist-processing-common==14.3.4; extra == "frozen"
 Requires-Dist: dkist-processing-core==7.3.0; extra == "frozen"
-Requires-Dist: dkist-processing-cryonirsp==1.19.19; extra == "frozen"
+Requires-Dist: dkist-processing-cryonirsp==2.0.0rc2; extra == "frozen"
 Requires-Dist: dkist-processing-math==2.2.3; extra == "frozen"
 Requires-Dist: dkist-processing-pac==3.1.2; extra == "frozen"
 Requires-Dist: dkist-service-configuration==4.3.0; extra == "frozen"
@@ -148,7 +148,7 @@ Requires-Dist: dkist_fits_specifications==4.28.0; extra == "frozen"
 Requires-Dist: dnspython==2.8.0; extra == "frozen"
 Requires-Dist: email-validator==2.3.0; extra == "frozen"
 Requires-Dist: executing==2.2.1; extra == "frozen"
-Requires-Dist: fastapi==0.138.0; extra == "frozen"
+Requires-Dist: fastapi==0.138.1; extra == "frozen"
 Requires-Dist: fastapi-cli==0.0.27; extra == "frozen"
 Requires-Dist: fastjsonschema==2.21.2; extra == "frozen"
 Requires-Dist: flower==2.0.1; extra == "frozen"
@@ -177,6 +177,7 @@ Requires-Dist: isoduration==20.11.0; extra == "frozen"
 Requires-Dist: itsdangerous==2.2.0; extra == "frozen"
 Requires-Dist: jedi==0.20.0; extra == "frozen"
 Requires-Dist: jmespath==1.1.0; extra == "frozen"
+Requires-Dist: joblib==1.5.3; extra == "frozen"
 Requires-Dist: json5==0.15.0; extra == "frozen"
 Requires-Dist: jsonpointer==3.1.1; extra == "frozen"
 Requires-Dist: jsonschema==4.26.0; extra == "frozen"
@@ -208,11 +209,12 @@ Requires-Dist: matplotlib==3.11.0; extra == "frozen"
 Requires-Dist: matplotlib-inline==0.2.2; extra == "frozen"
 Requires-Dist: mdurl==0.1.2; extra == "frozen"
 Requires-Dist: methodtools==0.4.7; extra == "frozen"
-Requires-Dist: mistune==3.3.1; extra == "frozen"
+Requires-Dist: mistune==3.3.2; extra == "frozen"
 Requires-Dist: more-itertools==11.1.0; extra == "frozen"
 Requires-Dist: moviepy==2.1.2; extra == "frozen"
 Requires-Dist: msgspec==0.21.1; extra == "frozen"
 Requires-Dist: multidict==6.7.1; extra == "frozen"
+Requires-Dist: narwhals==2.22.1; extra == "frozen"
 Requires-Dist: natsort==8.4.0; extra == "frozen"
 Requires-Dist: nbclient==0.11.0; extra == "frozen"
 Requires-Dist: nbconvert==7.17.1; extra == "frozen"
@@ -224,30 +226,30 @@ Requires-Dist: notebook_shim==0.2.4; extra == "frozen"
 Requires-Dist: numba==0.61.2; extra == "frozen"
 Requires-Dist: numpy==2.2.5; extra == "frozen"
 Requires-Dist: object-clerk==1.0.0; extra == "frozen"
-Requires-Dist: opentelemetry-api==1.42.1; extra == "frozen"
-Requires-Dist: opentelemetry-exporter-otlp==1.42.1; extra == "frozen"
-Requires-Dist: opentelemetry-exporter-otlp-proto-common==1.42.1; extra == "frozen"
-Requires-Dist: opentelemetry-exporter-otlp-proto-grpc==1.42.1; extra == "frozen"
-Requires-Dist: opentelemetry-exporter-otlp-proto-http==1.42.1; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation==0.63b1; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-aiohttp-client==0.63b1; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-asgi==0.63b1; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-botocore==0.63b1; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-celery==0.63b1; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-dbapi==0.63b1; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-fastapi==0.63b1; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-pika==0.63b1; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-psycopg2==0.63b1; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-pymongo==0.63b1; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-redis==0.63b1; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-requests==0.63b1; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-sqlalchemy==0.63b1; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-system-metrics==0.63b1; extra == "frozen"
+Requires-Dist: opentelemetry-api==1.43.0; extra == "frozen"
+Requires-Dist: opentelemetry-exporter-otlp==1.43.0; extra == "frozen"
+Requires-Dist: opentelemetry-exporter-otlp-proto-common==1.43.0; extra == "frozen"
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc==1.43.0; extra == "frozen"
+Requires-Dist: opentelemetry-exporter-otlp-proto-http==1.43.0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation==0.64b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-aiohttp-client==0.64b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-asgi==0.64b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-botocore==0.64b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-celery==0.64b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-dbapi==0.64b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-fastapi==0.64b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-pika==0.64b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-psycopg2==0.64b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-pymongo==0.64b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-redis==0.64b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-requests==0.64b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-sqlalchemy==0.64b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-system-metrics==0.64b0; extra == "frozen"
 Requires-Dist: opentelemetry-propagator-aws-xray==1.0.2; extra == "frozen"
-Requires-Dist: opentelemetry-proto==1.42.1; extra == "frozen"
-Requires-Dist: opentelemetry-sdk==1.42.1; extra == "frozen"
-Requires-Dist: opentelemetry-semantic-conventions==0.63b1; extra == "frozen"
-Requires-Dist: opentelemetry-util-http==0.63b1; extra == "frozen"
+Requires-Dist: opentelemetry-proto==1.43.0; extra == "frozen"
+Requires-Dist: opentelemetry-sdk==1.43.0; extra == "frozen"
+Requires-Dist: opentelemetry-semantic-conventions==0.64b0; extra == "frozen"
+Requires-Dist: opentelemetry-util-http==0.64b0; extra == "frozen"
 Requires-Dist: outcome==1.3.0.post0; extra == "frozen"
 Requires-Dist: packaging==26.2; extra == "frozen"
 Requires-Dist: pandas==3.0.3; extra == "frozen"
@@ -268,7 +270,7 @@ Requires-Dist: proglog==0.1.12; extra == "frozen"
 Requires-Dist: prometheus_client==0.25.0; extra == "frozen"
 Requires-Dist: prompt_toolkit==3.0.52; extra == "frozen"
 Requires-Dist: propcache==0.5.2; extra == "frozen"
-Requires-Dist: protobuf==6.33.6; extra == "frozen"
+Requires-Dist: protobuf==7.35.1; extra == "frozen"
 Requires-Dist: psutil==7.2.2; extra == "frozen"
 Requires-Dist: psycopg2-binary==2.9.12; extra == "frozen"
 Requires-Dist: ptyprocess==0.7.0; extra == "frozen"
@@ -301,6 +303,7 @@ Requires-Dist: rich-toolkit==0.20.1; extra == "frozen"
 Requires-Dist: rpds-py==2026.5.1; extra == "frozen"
 Requires-Dist: s3transfer==0.19.0; extra == "frozen"
 Requires-Dist: scikit-image==0.25.2; extra == "frozen"
+Requires-Dist: scikit-learn==1.9.0; extra == "frozen"
 Requires-Dist: scipy==1.15.3; extra == "frozen"
 Requires-Dist: semantic-version==2.10.0; extra == "frozen"
 Requires-Dist: setproctitle==1.3.7; extra == "frozen"
@@ -322,6 +325,7 @@ Requires-Dist: tenacity==8.5.0; extra == "frozen"
 Requires-Dist: termcolor==3.3.0; extra == "frozen"
 Requires-Dist: terminado==0.18.1; extra == "frozen"
 Requires-Dist: text-unidecode==1.3; extra == "frozen"
+Requires-Dist: threadpoolctl==3.6.0; extra == "frozen"
 Requires-Dist: tifffile==2026.6.1; extra == "frozen"
 Requires-Dist: tinycss2==1.5.1; extra == "frozen"
 Requires-Dist: tornado==6.5.7; extra == "frozen"

dkist_processing_cryonirsp-2.0.0rc2/changelog/289.bugfix.rst

@@ -0,0 +1,2 @@
+Use `resize_arrays_local_mean` to average arrays in the instrument polarization task instead of `resize_arrays`.
+With an output shape of `(1,1)`, `resize_arrays` returns the center pixel.

dkist_processing_cryonirsp-2.0.0rc2/changelog/292.feature.2.rst

@@ -0,0 +1,3 @@
+The calculation of a single spectral shift is now *only* via interpolation of the correlation signal between the reference
+and target spectra. Previously a second refinement of this value was done via a chi-squared minimization, but this was show
+to increase noise with no increase in accuracy.

dkist_processing_cryonirsp-2.0.0rc2/changelog/292.feature.rst

@@ -0,0 +1,2 @@
+Compute spectral shift calculations on smoothed/filtered spectral derivatives of the reference and target spectra. This
+amplifies the spectral signal and greatly improves the accuracy of the individual shift measurements.

dkist_processing_cryonirsp-2.0.0rc2/changelog/292.misc.rst

@@ -0,0 +1,3 @@
+Add option to all local trial workflows (``--parameter-doc``) that uses an existing input dataset parameter-part document
+to populate the parameters during a local run. This is an easy way for a user to run a pipeline locally using the exact
+parameters used in production.

dkist_processing_cryonirsp-2.0.0rc2/changelog/292.science.rst

@@ -0,0 +1,4 @@
+Improvements to the calculation of the spectral curvature to make them more robust against noisy input data. This change
+only affects science data for datasets where the spectral curvature was previously mis-identified. For these datasets this
+change will correct un-aligned spectral axes in L1 data and improve the quality of gain correction because the solar gain
+frames will have better removal of solar absorption lines.

{dkist_processing_cryonirsp-1.19.19 → dkist_processing_cryonirsp-2.0.0rc2}/dkist_processing_cryonirsp/models/parameters.py

@@ -37,11 +37,36 @@ class CryonirspParameters(ParameterBase, ParameterWavelengthMixin, ParameterArmI
         """Pixel precision (1/upsample_factor) to use during phase matching of beam/modulator images."""
         return self._find_most_recent_past_value("cryonirsp_geo_upsample_factor")
 
+    @property
+    def geo_savgol_window_length_px(self) -> int:
+        """Size of the window for the SavGol filter used to smooth the spectral derivative during prep for spectral shift measurements."""
+        return self._find_most_recent_past_value("cryonirsp_geo_savgol_window_length_px")
+
+    @property
+    def geo_savgol_polyorder(self) -> int:
+        """Order of the polynomial fit in the SavGol filter used to smooth the spectral derivative during prep for the spectral shift measurements."""
+        return self._find_most_recent_past_value("cryonirsp_geo_savgol_polyorder")
+
+    @property
+    def geo_derivative_cutoff_threshold(self) -> float:
+        """Threshold value for the spectral derivative above which pixels will be set to NaN."""
+        return self._find_most_recent_past_value("cryonirsp_geo_derivative_cutoff_threshold")
+
     @property
     def geo_max_shift(self) -> int:
         """Max allowed pixel shift when computing spectral curvature."""
         return self._find_most_recent_past_value("cryonirsp_geo_max_shift")
 
+    @property
+    def geo_shift_peak_interpolation_width_px(self) -> int:
+        """Return the ± pixel width used to interpolate the correlation peaks when computing spectral curvature."""
+        return self._find_most_recent_past_value("cryonirsp_geo_shift_peak_interpolation_width_px")
+
+    @property
+    def geo_poly_fit_min_samples(self) -> float:
+        """Return fractional number of samples required for the RANSAC regressor used to fit spectral shifts along the slit."""
+        return self._find_most_recent_past_value("cryonirsp_geo_poly_fit_min_samples")
+
     @property
     def geo_poly_fit_order(self) -> int:
         """Order of polynomial used to fit spectral shift as a function of slit position."""

{dkist_processing_cryonirsp-1.19.19 → dkist_processing_cryonirsp-2.0.0rc2}/dkist_processing_cryonirsp/tasks/instrument_polarization.py

@@ -12,6 +12,7 @@ from dkist_processing_math.arithmetic import divide_arrays_by_array
 from dkist_processing_math.arithmetic import subtract_array_from_arrays
 from dkist_processing_math.statistics import average_numpy_arrays
 from dkist_processing_math.transform.binning import resize_arrays
+from dkist_processing_math.transform.binning import resize_arrays_local_mean
 from dkist_processing_pac.fitter.polcal_fitter import PolcalFitter
 from dkist_processing_pac.input_data.drawer import Drawer
 from dkist_processing_pac.input_data.dresser import Dresser
@@ -281,8 +282,8 @@ class InstrumentPolarizationCalibrationBase(CryonirspTaskBase, ABC):
                 self.parameters.polcal_num_spatial_bins,
                 self.parameters.polcal_num_spectral_bins,
             )
-            local_binned_array = next(resize_arrays(gain_corrected_array, output_shape))
-            global_binned_array = next(resize_arrays(gain_corrected_array, (1, 1)))
+            local_binned_array = next(resize_arrays_local_mean(gain_corrected_array, output_shape))
+            global_binned_array = next(resize_arrays_local_mean(gain_corrected_array, (1, 1)))
 
         with self.telemetry_span(f"Create reduced CryonirspL0FitsAccess for {apm_str}"):
             local_result = CryonirspL0FitsAccess(

{dkist_processing_cryonirsp-1.19.19 → dkist_processing_cryonirsp-2.0.0rc2}/dkist_processing_cryonirsp/tasks/sp_geometric.py

@@ -4,14 +4,20 @@ import math
 
 import numpy as np
 import peakutils as pku
+import scipy.interpolate as spi
 import scipy.ndimage as spnd
+import scipy.signal as ss
 from dkist_processing_common.codecs.fits import fits_array_encoder
 from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_math.arithmetic import divide_arrays_by_array
 from dkist_processing_math.arithmetic import subtract_array_from_arrays
 from dkist_processing_math.statistics import average_numpy_arrays
 from dkist_service_configuration.logging import logger
-from scipy.optimize import minimize
+from sklearn.linear_model import RANSACRegressor
+from sklearn.pipeline import Pipeline
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.preprocessing import RobustScaler
 
 from dkist_processing_cryonirsp.codecs.fits import cryo_fits_array_decoder
 from dkist_processing_cryonirsp.models.beam_boundaries import BeamBoundary
@@ -97,6 +103,9 @@ class SPGeometricCalibration(CryonirspTaskBase, ShiftMeasurementsMixin):
                         beam=beam,
                     )
 
+                with self.telemetry_span(f"Preparing beam {beam} for spectral shift measurement"):
+                    self.prepare_array_for_shift_measurement(beam=beam)
+
                 with self.telemetry_span(f"Computing spectral shifts for {beam = }"):
                     spec_shifts = self.compute_spectral_shifts(beam=beam)
 
@@ -181,6 +190,22 @@ class SPGeometricCalibration(CryonirspTaskBase, ShiftMeasurementsMixin):
         )
         return average_numpy_arrays(array_generator)
 
+    def shift_prepared_data(self, beam: int) -> np.ndarray:
+        """
+        Array for a single beam that has been prepared for measurements of spectral shift.
+
+        I.e., the array created by `prepare_array_for_spectral_shift`
+        """
+        return next(
+            self.read(
+                tags=[
+                    CryonirspTag.intermediate_frame(beam=beam),
+                    CryonirspTag.task("GC_SHIFT_PREPARED"),
+                ],
+                decoder=cryo_fits_array_decoder,
+            )
+        )
+
     def do_basic_corrections(self):
         """Apply dark, bad pixel and lamp gain corrections to all data that will be used for Geometric Calibration."""
         # There is likely only a single exposure conditions tuple in the list, but we iterate over the list
@@ -415,11 +440,11 @@ class SPGeometricCalibration(CryonirspTaskBase, ShiftMeasurementsMixin):
         I.e., the spectral shift at each slit position needed to have wavelength be constant across a single spatial
         pixel. Generally, the algorithm is:
 
-         1. Identify the reference array spectrum as the center of the slit
-         2. For each slit position, make an initial guess of the shift via correlation
-         3. Take the initial guesses and use them in a chisq minimizer to refine the shifts
-         4. Interpolate over those shifts identified as too large
-         5. Remove the mean shift so the total shift amount is minimized
+        #. Identify the reference array spectrum as the center of the slit
+        #. For each slit position, measure the shift by cross-correlating the spectrum with the reference spectrum
+        #. Mask spatial pixels with large RMS differences between the reference spectrum and their shifted spectra
+        #. Fit a polynomial to the measured shifts as a function of slit position
+        #. Remove the mean shift so the total shift amount is minimized
 
         Parameters
         ----------
@@ -432,32 +457,39 @@ class SPGeometricCalibration(CryonirspTaskBase, ShiftMeasurementsMixin):
             Spectral shift for a single beam
         """
         logger.info(f"Computing spectral shifts for beam {beam}")
-        beam_array = self.offset_corrected_data(beam=beam)
+        beam_array = self.shift_prepared_data(beam=beam)
         spatial_size = beam_array.shape[0]
 
         if beam == 1:
             # Use the same reference spectrum for both beams.
             # We pick the spectrum from the center of the slit, with a buffer of 10 pixels on either side
             middle_row = spatial_size // 2
-            self.ref_spec = np.nanmedian(beam_array[middle_row - 10 : middle_row + 10, :], axis=0)
+            raw_ref_spec = np.nanmedian(beam_array[middle_row - 10 : middle_row + 10, :], axis=0)
+            cleaned_ref_spec = self.clean_spectrum(spectrum=raw_ref_spec)
+            self.ref_spec = cleaned_ref_spec
 
-        beam_shifts = np.empty(spatial_size) * np.nan
+        beam_shifts = np.full(spatial_size, np.nan)
+        beam_rms = np.full(spatial_size, np.nan)
         for i in range(spatial_size):
-            target_spec = beam_array[i, :]
+            target_spec = self.clean_spectrum(spectrum=beam_array[i, :])
 
-            initial_guess = self.compute_initial_spec_shift_guess(
+            shift = self.compute_single_spec_shift(
                 ref_spec=self.ref_spec, target_spec=target_spec, beam=beam, pos=i
             )
 
-            shift = self.compute_single_spec_shift(
-                ref_spec=self.ref_spec,
-                target_spec=target_spec,
-                initial_guess=initial_guess,
-                beam=beam,
-                pos=i,
-            )
+            shifted_target = spnd.shift(target_spec, -shift, cval=np.nan)
+            rms = np.nanstd(self.ref_spec - shifted_target)
 
             beam_shifts[i] = shift
+            beam_rms[i] = rms
+
+        # Mask spatial positions with large residuals even after applying the measured shift
+        # Setting the threshold as 4 times the second quartile amount is a good way to estimate are large
+        # fraction of the CDF from a region that won't be polluted with outliers.
+        rms_median, rms_25_percentile = np.nanpercentile(beam_rms, [50, 25])
+        rms_threshold = rms_median + 4 * (rms_median - rms_25_percentile)
+        logger.info(f"Raw shift RMS threshold for beam {beam} is {rms_threshold}")
+        beam_shifts[beam_rms > rms_threshold] = np.nan
 
         # Subtract the average so we shift my a minimal amount
         if beam == 1:
@@ -477,22 +509,95 @@ class SPGeometricCalibration(CryonirspTaskBase, ShiftMeasurementsMixin):
 
         # Finally, fit the shifts and return the resulting polynomial. Any "bad" fits were set to NaN and will be
         # interpolated over.
-        poly_fit_order = self.parameters.geo_poly_fit_order
         nan_idx = np.isnan(beam_shifts)
-        poly = np.poly1d(
-            np.polyfit(np.arange(spatial_size)[~nan_idx], beam_shifts[~nan_idx], poly_fit_order)
+        ransac_abscissa = np.arange(spatial_size)[:, None]
+        model = self.build_RANSAC_model()
+        model.fit(ransac_abscissa[~nan_idx, :], beam_shifts[~nan_idx])
+
+        return model.predict(ransac_abscissa)
+
+    def prepare_array_for_shift_measurement(self, beam: int) -> None:
+        """
+        Apply filtering to an array to amplify the spectral signal and improve subsequent shift measurements.
+
+        The array to be filtered is the fully offset-and-angle corrected array for the given beam.
+
+        The filtering steps are:
+
+        #. Normalize each spectral pixel by a fixed value of its CDF. This removes scaling differences between the reference and target spectra.
+
+           a. At this stage we also clip values outside the range [0, 2]
+
+        #. Computed a smoothed spectral derivative for each spatial pixel. This step greatly increases the strength of the true spectral signal.
+
+        #. Apply a spectral windowing function to protect the correlation of the reference and target spectra from weird and wild edge-effects.
+        """
+        beam_array = self.offset_corrected_data(beam=beam)
+
+        normalized = (
+            beam_array
+            / np.nanpercentile(
+                beam_array,
+                self.parameters.solar_characteristic_spatial_normalization_percentile,
+                axis=1,
+            )[:, None]
+        )
+
+        # The normalized array really shouldn't have any values less than 0 or more than 2; we're essentially
+        # saying we expect the dynamic range of the spectrum to be less than 200% of the continuum value, which
+        # is a very reasonable assumption.
+        clipped = np.clip(normalized, 0, 2)
+
+        # Compute a smoothed derivative in one step with the `deriv` kwarg to `savgol_filter`
+        smoothed_derivative = ss.savgol_filter(
+            clipped,
+            window_length=self.parameters.geo_savgol_window_length_px,
+            polyorder=self.parameters.geo_savgol_polyorder,
+            deriv=1,
+            axis=1,
         )
 
-        return poly(np.arange(spatial_size))
+        # Apply a window to reduce bad edge signals. This works because the mean of the derivative should be 0.
+        windowed = smoothed_derivative * ss.windows.tukey(smoothed_derivative.shape[1])[None, :]
+        windowed[np.abs(windowed) > self.parameters.geo_derivative_cutoff_threshold] = np.nan
 
-    def compute_initial_spec_shift_guess(
+        self.write(
+            data=windowed,
+            tags=[
+                CryonirspTag.intermediate_frame(beam=beam),
+                CryonirspTag.task("GC_SHIFT_PREPARED"),
+            ],
+            encoder=fits_array_encoder,
+        )
+
+    def clean_spectrum(self, spectrum: np.ndarray) -> np.ndarray:
+        """
+        Clean non-finite pixels in a spectrum by interpolating over them.
+
+        Interpolation is done via nearest-neighbor and any extrapolated values are set to the mean of the
+        spectrum so that they don't contribute any signal to `compute_single_spec_shift`.
+        """
+        bad_mask = ~np.isfinite(spectrum)
+        abscissa = np.arange(spectrum.size)
+        interp_func = spi.interp1d(
+            abscissa[~bad_mask],
+            spectrum[~bad_mask],
+            kind="nearest",
+            bounds_error=False,
+            fill_value=np.mean(spectrum[~bad_mask]),
+        )
+        spectrum[bad_mask] = interp_func(abscissa[bad_mask])
+        return spectrum
+
+    def compute_single_spec_shift(
         self, *, ref_spec: np.ndarray, target_spec: np.ndarray, beam: int, pos: int
     ) -> float:
         """
-        Make a rough guess for the offset between two spectra.
+        Compute the offset between two spectra via correlation.
 
-        A basic correlation is performed and the location of the peak sets the initial guess. If more than one strong
-        peak is found then the peak locations are averaged together.
+        A basic correlation is performed and the location of the correlation peak is the shift. The true peak is found
+        by interpolating around the maximum correlation location. If more than one strong peak is found then the peak
+        locations are averaged together.
         """
         corr = np.correlate(
             target_spec - np.nanmean(target_spec),
@@ -507,84 +612,59 @@ class SPGeometricCalibration(CryonirspTaskBase, ShiftMeasurementsMixin):
         truncated_corr = corr[start:stop]
 
         # This min_dist ensures we only find a single peak in each correlation signal
-        pidx = pku.indexes(truncated_corr, min_dist=truncated_corr.size)
-        initial_guess = 1 * (pidx - truncated_corr.size // 2)
+        peak_idx = pku.indexes(truncated_corr, min_dist=truncated_corr.size)
 
-        # These edge-cases are very rare, but do happen sometimes
-        if initial_guess.size == 0:
+        # This edge-case is very rare, but does happen sometimes
+        if peak_idx.size != 1:
             logger.info(
-                f"Spatial position {pos} in {beam=} doesn't have a correlation peak. Initial guess set to 0"
+                f"Spatial position {pos} in {beam = } has either no or multiple correlation peaks. Shift set to NaN."
             )
-            initial_guess = 0.0
-
-        elif initial_guess.size > 1:
+            return np.nan
+
+        try:
+            peak_interp = pku.interpolate(
+                np.arange(truncated_corr.size),
+                truncated_corr,
+                ind=peak_idx,
+                width=self.parameters.geo_shift_peak_interpolation_width_px,
+            )
+        except:
             logger.info(
-                f"Spatial position {pos} in {beam=} has more than one correlation peak ({initial_guess}). Initial guess set to mean ({np.nanmean(initial_guess)})"
+                f"Spatial position {pos} in {beam = } failed to interpolate the correlation peak. Falling back "
+                "to the rough peak."
             )
-            initial_guess = np.nanmean(initial_guess)
+            peak_interp = peak_idx[0]
 
-        return initial_guess
+        shift = 1 * (peak_interp - truncated_corr.size // 2)
 
-    def compute_single_spec_shift(
-        self,
-        *,
-        ref_spec: np.ndarray,
-        target_spec: np.ndarray,
-        initial_guess: float,
-        beam: int,
-        pos: int,
-    ) -> float:
-        """
-        Refine the 1D offset between two spectra.
+        return shift.item(0)
 
-        A 1-parameter minimization is performed where the goodness-of-fit parameter is simply the Chisq difference
-        between the reference spectrum and shifted target spectrum.
+    def build_RANSAC_model(self) -> Pipeline:
         """
-        shift = minimize(
-            self.shift_chisq,
-            np.atleast_1d(initial_guess),
-            args=(ref_spec, target_spec),
-            method="nelder-mead",
-        ).x[0]
-
-        max_shift = self.parameters.geo_max_shift
-        if np.abs(shift) > max_shift:
-            # Didn't find a good peak
-            logger.info(
-                f"shift in {beam = } at spatial pixel {pos} out of range ({shift} > {max_shift})"
-            )
-            shift = np.nan
+        Build a scikit-learn pipeline from a set of estimators.
 
-        return shift
+        Namely, construct a `~sklearn.pipeline.Pipeline` built from
 
-    @staticmethod
-    def shift_chisq(par: np.ndarray, ref_spec: np.ndarray, spec: np.ndarray) -> float:
+        `~sklearn.preprocessing.PolynomialFeatures` -> `~sklearn.preprocessing.RobustScaler` -> `~sklearn.linear_model.RANSACRegressor`
         """
-        Goodness of fit calculation for a simple shift. Uses simple chisq as goodness of fit.
+        fit_order = self.parameters.geo_poly_fit_order
+        min_samples = self.parameters.geo_poly_fit_min_samples
+        logger.info(f"Building RANSAC model with {fit_order = } and {min_samples = }")
 
-        Less robust than SPGainCalibration's `refine_shift`, but much faster.
+        # PolynomialFeatures casts the pipeline as a polynomial fit
+        # see https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.PolynomialFeatures.html#sklearn.preprocessing.PolynomialFeatures
+        poly_feature = PolynomialFeatures(degree=fit_order)
 
-        Parameters
-        ----------
-        par : np.ndarray
-            Spectral shift being optimized
+        # RobustScaler is a scale factor that is robust to outliers
+        # see https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.RobustScaler.html#sklearn.preprocessing.RobustScaler
+        scaler = RobustScaler()
 
-        ref_spec : np.ndarray
-            Reference spectra
+        # The RANSAC regressor iteratively sub-samples the input data and constructs a model with this sub-sample.
+        # The method used allows it to be robust to outliers.
+        # see https://scikit-learn.org/stable/modules/linear_model.html#ransac-regression
+        RANSAC = RANSACRegressor(min_samples=min_samples)
 
-        spec : np.ndarray
-            Spectra being fitted
-
-        Returns
-        -------
-        float
-            Sum of chisquared fit
-
-        """
-        shift = par[0]
-        shifted_spec = spnd.shift(spec, -shift, mode="constant", cval=np.nan)
-        chisq = np.nansum((ref_spec - shifted_spec) ** 2 / ref_spec)
-        return chisq
+        return make_pipeline(poly_feature, scaler, RANSAC)
 
     def write_angle(self, angle: float, beam: int) -> None:
         """

{dkist_processing_cryonirsp-1.19.19 → dkist_processing_cryonirsp-2.0.0rc2}/dkist_processing_cryonirsp/tests/conftest.py

@@ -390,8 +390,13 @@ def cryonirsp_testing_parameters_factory(
         cryonirsp_polcal_num_spectral_bins: int = 1
         cryonirsp_polcal_pac_fit_mode: str = "use_M12_I_sys_per_step"
         cryonirsp_geo_upsample_factor: int = 100
+        cryonirsp_geo_savgol_window_length_px: int = 15
+        cryonirsp_geo_savgol_polyorder: int = 3
+        cryonirsp_geo_derivative_cutoff_threshold: float = 0.2
         cryonirsp_geo_max_shift: int = 80
-        cryonirsp_geo_poly_fit_order: int = 3
+        cryonirsp_geo_shift_peak_interpolation_width_px: int = 5
+        cryonirsp_geo_poly_fit_min_samples: float = 0.6
+        cryonirsp_geo_poly_fit_order: int = 4
         cryonirsp_geo_long_axis_gradient_displacement: int = 4
         cryonirsp_geo_strip_long_axis_size_fraction: float = 0.8
         cryonirsp_geo_strip_short_axis_size_fraction: float = 0.1