dkist-processing-trend 0.1.0rc1__py3-none-any.whl

dkist_processing_trend/tasks/visp/visp_geometric.py

@@ -0,0 +1,260 @@
+"""Task for computing the ViSP beam offset."""
+
+from typing import Literal
+
+import numpy as np
+import peakutils as pku
+import scipy.ndimage as spnd
+from astropy.modeling import fitting
+from astropy.modeling import models
+from astropy.stats import sigma_clip
+from dkist_processing_common.codecs.asdf import asdf_encoder
+from dkist_processing_common.codecs.fits import fits_array_decoder
+from dkist_processing_math.statistics import average_numpy_arrays
+from dkist_service_configuration.logging import logger
+from skimage.registration import phase_cross_correlation
+
+from dkist_processing_trend.models.tags import TrendTag
+from dkist_processing_trend.tasks.trend_base import TrendArmTaskBase
+from dkist_processing_trend.tasks.visp.visp_extract_beam import extract_visp_beam
+
+__all__ = ["VispGeometricCalibration"]
+
+
+class VispGeometricCalibration(TrendArmTaskBase):
+    """
+    Compute the X/Y offset between the two ViSP beams.
+
+    Unlike the main L1 pipeline, this task does not treat each modstate separately and does not compute a rotational offset.
+    For the purposes of the Trend pipeline a simple X/Y shift is sufficient.
+
+    Parameters
+    ----------
+    arm_id
+        id of the instrument arm to operate on
+
+    recipe_run_id
+        id of the recipe run used to identify the workflow run this task is part of
+
+    workflow_name
+        name of the workflow to which this instance of the task belongs
+
+    workflow_version
+        version of the workflow to which this instance of the task belongs
+    """
+
+    record_provenance = True
+
+    def run(self):
+        """
+        Compute the X/Y shift between ViSP beam 1 and 2.
+
+        Algorithm:
+
+        #. Compute the average of all input POLCAL_GAIN frames (i.e., "clear" CS step)
+        #. Split this average into beams 1 and 2
+        #. Compute the rotation angle for each beam
+        #. Correct each beam for its measured rotation angle
+        #. Collapse each beam along both the spectral and spatial axis and  use `~skimage.registration.phase_cross_correlation` to compute the shift along each axis
+        #. Write the shifts and angles to scratch
+
+        NOTE: The outputs of this task are angles and shifts that should be used *directly* with the geometric correction
+        method; no inverting/sign flipping is needed.
+        """
+        with self.telemetry_span("Compute average clear array"):
+            logger.info("Computing average clear array")
+            full_avg_array = self.compute_avg_clear_array()
+
+        logger.info("Extracting beams")
+        beam_border = self.parameters.visp_beam_border
+        beam1_array = extract_visp_beam(full_avg_array, beam=1, beam_border=beam_border)
+        beam2_array = extract_visp_beam(full_avg_array, beam=2, beam_border=beam_border)
+
+        with self.telemetry_span("Compute beam angles"):
+            logger.info("Computing beam angles")
+            beam1_angle_rad = self.compute_beam_angle(array=beam1_array)
+            beam1_angle_deg = np.rad2deg(beam1_angle_rad)
+            logger.info(f"Beam 1 angle = {beam1_angle_deg:.5f} deg")
+
+            beam2_angle_rad = self.compute_beam_angle(array=beam2_array)
+            beam2_angle_deg = np.rad2deg(beam2_angle_rad)
+            logger.info(f"Beam 2 angle = {beam2_angle_deg:.5f} deg")
+
+        with self.telemetry_span("Compute beam offset"):
+            logger.info("Computing beam offsets")
+
+            logger.info("Removing beam angles")
+            beam1_rotated = spnd.rotate(
+                beam1_array, angle=-beam1_angle_deg, reshape=False, cval=np.nanmedian(beam1_array)
+            )
+            beam2_rotated = spnd.rotate(
+                beam2_array, angle=-beam2_angle_deg, reshape=False, cval=np.nanmedian(beam2_array)
+            )
+
+            logger.info("Filtering beams")
+            beam1_filtered = self.high_pass_filter_array(beam1_rotated)
+            beam2_filtered = self.high_pass_filter_array(beam2_rotated)
+
+            shift = np.full(2, np.nan)
+            for i, (ax_name, ax) in enumerate(zip(["spectral", "spatial"], [0, 1])):
+                logger.info(f"Computing {ax_name} shift")
+                shift[i] = self.compute_single_axis_shift(
+                    reference_array=beam1_filtered, target_array=beam2_filtered, axis=ax
+                )
+
+            logger.info(f"Measured {shift = }")
+
+        logger.info("Writing angles and shift to scratch")
+        beam1_geo_data = {"angle_rad": beam1_angle_rad, "shift": np.array([0, 0.0])}
+        self.write(
+            data=beam1_geo_data,
+            tags=[
+                TrendTag.intermediate(),
+                TrendTag.arm_id(self.arm_id),
+                TrendTag.beam(1),
+                TrendTag.task_visp_geometric_calibration(),
+            ],
+            encoder=asdf_encoder,
+        )
+
+        beam2_geo_data = {
+            "angle_rad": beam2_angle_rad,
+            "shift": -shift,
+        }
+        self.write(
+            data=beam2_geo_data,
+            tags=[
+                TrendTag.intermediate(),
+                TrendTag.arm_id(self.arm_id),
+                TrendTag.beam(2),
+                TrendTag.task_visp_geometric_calibration(),
+            ],
+            encoder=asdf_encoder,
+        )
+
+    def compute_avg_clear_array(self) -> np.ndarray:
+        """Compute the average of all "clear" CS step arrays."""
+        tags = [TrendTag.input(), TrendTag.arm_id(self.arm_id), TrendTag.task_polcal_gain()]
+        logger.info(f"Using {self.count(tags)} INPUT clear arrays.")
+        clear_arrays = self.read(tags=tags, decoder=fits_array_decoder)
+        avg_array = average_numpy_arrays(clear_arrays)
+        return avg_array
+
+    def compute_beam_angle(self, array: np.ndarray) -> float:
+        """
+        Compute the rotation angle of a single beam.
+
+        The angle is computed by measuring the slope of the slit hairlines; the arctan of the slope then gives the rotation angle.
+        To measure the slope of the hairlines, first the hairline signal is isolated with by smoothing the image in the spatial
+        direction (which will remove the hairlines) and then subtracting this smoothed array from the input image.
+        Once the hairlines have been isolated, the location of hairline center is measured with `peakutils <https://pypi.org/project/PeakUtils/>`_.
+        Finally, a line is fit to the the location of hairline center as a function of wavelength and the slope of this line is
+        used to find the angle.
+
+        Multiple hairlines (always 2, and sometimes more if there are reflections) have their angles computed separately
+        and then average together for the final result.
+
+        Returns
+        -------
+        The rotation angle of the given array, in radians.
+        """
+        no_hairline_array = spnd.median_filter(array, size=(1, 30))
+        hairline_only_array = (array - no_hairline_array) / no_hairline_array
+
+        # Turn low-signal hairlines into "emission" lines so peakutils can find them
+        hairline_only_array *= -1
+
+        hairline_1d = np.nanmedian(hairline_only_array, axis=0)
+        hairline_locs = pku.indexes(hairline_1d, thres=0.8)
+
+        num_wave, num_spat = array.shape
+        fit_centers = np.full((len(hairline_locs), num_wave), np.nan)
+        spectral_abscissa = np.arange(num_wave)
+        spatial_abscissa = np.arange(num_spat)
+        for i in range(num_wave):
+            try:
+                interp_center = pku.interpolate(
+                    spatial_abscissa, hairline_only_array[i, :], ind=hairline_locs, width=10
+                )
+                fit_centers[:, i] = interp_center
+            except:
+                logger.info(f"Failed to fit hairline center for spectral pixel {i}. Ignoring.")
+
+        hairline_angles = []
+        for hairline in range(fit_centers.shape[0]):
+            fit_angle = self.fit_angle_from_slope(
+                abscissa=spectral_abscissa, centers=fit_centers[hairline]
+            )
+            hairline_angles.append(fit_angle)
+
+        return np.mean(hairline_angles)
+
+    @staticmethod
+    def fit_angle_from_slope(abscissa: np.ndarray, centers: np.ndarray, sigma: float = 3) -> float:
+        """
+        Fit a 1-degree polynomial (i.e., a line) to the hairline centers and compute an angle from the slope.
+
+        Outliers are iteratively removed using a sigma clipping algorithm as detailed here:
+        https://docs.astropy.org/en/stable/modeling/example-fitting-line.html#iterative-fitting-using-sigma-clipping
+
+        Returns
+        -------
+        The slope of the line, in radians.
+        """
+        line_fitter = fitting.LinearLSQFitter()
+        outlier_rejection_fitter = fitting.FittingWithOutlierRemoval(
+            fitter=line_fitter,
+            outlier_func=sigma_clip,
+            sigma=sigma,
+            cenfunc="median",
+            stdfunc="std",
+        )
+        linear_model = models.Linear1D()
+
+        fit_line = outlier_rejection_fitter(model=linear_model, x=abscissa, y=centers)[0]
+        angle_rad = np.arctan(fit_line.slope.value)
+        return angle_rad
+
+    @staticmethod
+    def high_pass_filter_array(array: np.ndarray) -> np.ndarray:
+        """
+        Remove low-frequency signal in an array by dividing it by a smoothed copy.
+
+        The smoothing is done with a Gaussian smoothing kernel. The resulting array has strong spectral features
+        and hairlines accentuated over the continuum.
+        """
+        # The sigma value is hardcoded in the ViSP L1 pipeline, too. It's valid for all ViSP data for all time.
+        return array / spnd.gaussian_filter(array, sigma=5)
+
+    def compute_single_axis_shift(
+        self, reference_array: np.ndarray, target_array: np.ndarray, axis: Literal[0, 1]
+    ) -> float:
+        """
+        Compute the offset between two arrays along a single axis.
+
+        ViSP data have strong, high-frequency signals along both of their axes; the solar spectrum and the slit hairlines.
+        This method isolates one of those signals by first computing a derivative along the axis of interest. Because
+        the two signals are orthogonal this derivative both accentuates the strong features along the axis of interest and
+        removes the signal in the orthogonal direction (e.g., the hairlines are constant along the spectral axis).
+
+        The resulting derivative is then collapsed along the orthogonal axis, via median, into a single vector, which
+        increases the signal to noise and simplifies the final correlation.
+
+        The single vectors from both arrays are then registered via `~skimage.registration.phase_cross_correlation` to
+        produce the final shift.
+        """
+        opposite_axis = abs(axis - 1)
+
+        reference_deriv = np.diff(reference_array, axis=axis)
+        reference_1d_signal = np.nanmedian(reference_deriv, axis=opposite_axis)
+
+        target_deriv = np.diff(target_array, axis=axis)
+        target_1d_signal = np.nanmedian(target_deriv, axis=opposite_axis)
+
+        shift = phase_cross_correlation(
+            reference_image=reference_1d_signal,
+            moving_image=target_1d_signal,
+            upsample_factor=self.parameters.visp_geo_upsample_factor,
+            normalization=None,
+        )[0]
+        return shift.item(0)

dkist_processing_trend/tasks/visp/visp_prep_fit_data.py

@@ -0,0 +1,162 @@
+"""Task for preparing ViSP data for fitting by `dkist_processing_pac`."""
+
+import numpy as np
+import scipy.ndimage as spnd
+
+from dkist_processing_trend.models.instrument_options import VispInstrumentOptions
+from dkist_processing_trend.tasks.prepare_fit_data_base import PrepareFitDataBase
+from dkist_processing_trend.tasks.visp.visp_extract_beam import extract_visp_beam
+
+__all__ = ["VispPrepareFitData"]
+
+
+class VispPrepareFitData(PrepareFitDataBase):
+    """
+    Subclass of `~dkist_processing_trend.tasks.prepare_fit_data_base.PrepareFitDataBase` that adds functionality for working with ViSP data.
+
+    Parameters
+    ----------
+    arm_id
+        id of the instrument arm to operate on
+
+    recipe_run_id
+        id of the recipe run used to identify the workflow run this task is part of
+
+    workflow_name
+        name of the workflow to which this instance of the task belongs
+
+    workflow_version
+        version of the workflow to which this instance of the task belongs
+    """
+
+    def extract_beam(self, array: np.ndarray, beam: int) -> np.ndarray:
+        """Extract a given beam array from a full ViSP frame."""
+        beam_border = self.parameters.visp_beam_border
+        return extract_visp_beam(array=array, beam=beam, beam_border=beam_border)
+
+    def apply_global_instrument_options(
+        self, array: np.ndarray, instrument_options: VispInstrumentOptions
+    ) -> np.ndarray:
+        """
+        Prepare global ViSP data for PAC fitting.
+
+        We first mask the slit hairlines and then simply compute a global median of the array.
+        """
+        filtered_array = self.mask_hairlines(array)
+
+        return np.nanmedian(filtered_array)[None, None]
+
+    def apply_local_instrument_options(
+        self, array: np.ndarray, instrument_options: VispInstrumentOptions
+    ) -> np.ndarray:
+        """
+        Prepare local ViSP data for PAC fitting.
+
+        Algorithm:
+
+        #. Mask the slit hairlines
+        #. Collapse the array by computing the median along the spectral dimension
+        #. Smooth the result in the spatial dimension
+        #. Bin the array in the spatial dimension using the local median to compute the value of each bin
+        """
+        filtered_array = self.mask_hairlines(array)
+
+        # Add back in a dummy spectral dimension so things stay 2D, which helps thinking about it later on.
+        spectral_binned_array = np.nanmedian(filtered_array, axis=0)[None, :]
+
+        spatially_smoothed_array = spnd.median_filter(
+            spectral_binned_array,
+            # The 1 below means we don't smooth in the spectral dimension
+            size=(1, self.parameters.visp_polcal_spatial_median_filter_width_px),
+        )
+
+        local_array = self.downsample_spatial_dimension_local_median(
+            spatially_smoothed_array, instrument_options.num_spatial_px
+        )
+
+        return local_array
+
+    def mask_hairlines(self, array: np.ndarray) -> np.ndarray:
+        # This method is copied directly from the `CorrectionsMixin` in `dkist-processing-visp`
+        """
+        Mask hairlines from an array.
+
+        The hairlines will be replaced with data from a median-filtered version of the input array.
+
+        Hairlines are identified by first subtracting a spatially smoothed copy of the array and then looking for pixels
+        that have large differences. This works because the hairlines are the only features that are sharp in the
+        spatial dimension. The identified hairlines are then slightly smoothed spatially to ensure that their
+        higher-flux wings are correctly masked.
+        """
+        filtered_array = self.median_filter_array_for_hairline_identification(array)
+        hairline_locations = self.find_hairline_pixels(
+            input_array=array, filtered_array=filtered_array
+        )
+
+        # Replace hairline pixels with data from the spatially-filtered array
+        array[hairline_locations] = filtered_array[hairline_locations]
+
+        return array
+
+    def median_filter_array_for_hairline_identification(self, array: np.ndarray) -> np.ndarray:
+        # This method is copied directly from the `CorrectionsMixin` in `dkist-processing-visp`
+        """
+        Small helper to separate out the median filter step of hairline identification.
+
+        This step has been factored out so that functions that need the filtered array for further processing can avoid
+        repeating this expensive computation.
+        """
+        # The size=(1, X) means we only smooth in the spatial dimension (1st axis)
+        filtered_array = spnd.median_filter(
+            array, size=(1, self.parameters.visp_hairline_median_spatial_smoothing_width_px)
+        )
+        return filtered_array
+
+    def find_hairline_pixels(
+        self, input_array: np.ndarray, filtered_array: np.ndarray
+    ) -> np.ndarray:
+        # This method is copied directly from the `CorrectionsMixin` in `dkist-processing-visp`
+        """
+        Find pixels that likely correspond to hairlines.
+
+        This also slightly smooths the identified pixels so that high-flux wings of the hairlines are included.
+        """
+        diff = (input_array - filtered_array) / filtered_array
+        hairline_locations = np.abs(diff) > self.parameters.visp_hairline_fraction
+
+        # Now smooth the hairline mask in the spatial dimension to capture the higher-flux wings
+        mask_array = np.zeros_like(input_array)
+        mask_array[hairline_locations] = 1.0
+        mask_array = spnd.gaussian_filter1d(
+            mask_array, self.parameters.visp_hairline_mask_spatial_smoothing_width_px, axis=1
+        )
+
+        hairline_locations = np.where(
+            mask_array
+            > mask_array.max() * self.parameters.visp_hairline_mask_gaussian_peak_cutoff_fraction
+        )
+
+        return hairline_locations
+
+    @staticmethod
+    def downsample_spatial_dimension_local_median(
+        data: np.ndarray, num_spatial_bins: int
+    ) -> np.ndarray:
+        # Copied directly from the `DownsampleMixin` of `dkist-processing-visp`
+        """Resample a stack of spectra along the spatial dimension.
+
+        This is a separate function because calling `skimage.measure.block_reduce` on the entire (large) array all at once
+        creates a huge memory strain that is not needed since we're only reducing over one of the dimensions. Instead,
+        this function does some very cool tricks with reshaping and base numpy to keep the memory footprint lower.
+        """
+        # Taken from the amazing answer in
+        #  https://stackoverflow.com/questions/44527579/whats-the-best-way-to-downsample-a-numpy-array
+        num_wave, num_spat_pos = data.shape
+
+        if (num_spat_pos / num_spatial_bins) % 1 != 0:
+            raise ValueError(
+                f"The number of spatial bins must evenly divide the spatial dimension. {num_spatial_bins} bins do not evenly divide {num_spat_pos}."
+            )
+
+        reshaped = data.reshape((num_wave, num_spatial_bins, num_spat_pos // num_spatial_bins))
+        return np.median(reshaped, axis=2)

dkist_processing_trend/tasks/visp/visp_process_demod.py

@@ -0,0 +1,236 @@
+"""Task for producing full-frame ViSP demodulation matrices from the output of PAC fitting."""
+
+from typing import Literal
+
+import numpy as np
+from dkist_processing_common.codecs.fits import fits_array_decoder
+from dkist_processing_common.codecs.fits import fits_array_encoder
+from dkist_processing_math.transform.binning import resize_arrays
+from dkist_service_configuration.logging import logger
+from sklearn.linear_model import RANSACRegressor
+from sklearn.pipeline import Pipeline
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.preprocessing import RobustScaler
+
+from dkist_processing_trend.models.tags import TrendTag
+from dkist_processing_trend.tasks.trend_base import TrendArmTaskBase
+from dkist_processing_trend.tasks.visp.visp_extract_beam import extract_visp_beam
+
+__all__ = ["VispProcessDemodulationMatrices"]
+
+
+class VispProcessDemodulationMatrices(TrendArmTaskBase):
+    """
+    Task class for producing full-frame demodulation ViSP matrices from the output of PAC fits.
+
+    This task mimics the functionality (and code) of the `InstrumentPolarizationCalibration <https://docs.dkist.nso.edu/projects/visp/en/stable/autoapi/dkist_processing_visp/tasks/instrument_polarization/index.html>`_
+    in the L1 ViSP pipeline; it first smooths the fit modulation matrices in the spatial dimension before upsampling to
+    the full-frame shape.
+
+    Parameters
+    ----------
+    arm_id
+        id of the instrument arm to operate on
+
+    recipe_run_id
+        id of the recipe run used to identify the workflow run this task is part of
+
+    workflow_name
+        name of the workflow to which this instance of the task belongs
+
+    workflow_version
+        version of the workflow to which this instance of the task belongs
+    """
+
+    record_provenance = True
+
+    def run(self):
+        """
+        Smooth and upsample raw best-fit demodulation matrices to match the full ViSP beam size.
+
+        First, the MODulation matrices are smoothed element-by-element along the spatial dimension by fitting with a polynomial.
+        This also has the effect of upsampling the spatial dimension to the full-beam size.
+        Finally, the wavelength dimension is upsampled to the full-beam size via interpolation.
+        """
+        for beam in [1, 2]:
+            full_beam_shape = self.get_full_beam_shape(beam)
+            logger.info(f"{full_beam_shape = }")
+            for fit_options in self.parameters.fit_options_list:
+                for instrument_options in self.parameters.instrument_processing_options:
+                    log_str = f"fit options '{fit_options.name}', inst options '{instrument_options.name}', and {beam = }"
+                    base_tags = [
+                        TrendTag.intermediate(),
+                        TrendTag.pac_fit_options(fit_options.name),
+                        TrendTag.instrument_processing_options(instrument_options.name),
+                        TrendTag.beam(beam),
+                        TrendTag.arm_id(self.arm_id),
+                    ]
+                    with self.telemetry_span(f"Resampling demodulation matrices for {log_str}"):
+                        raw_demod_matrices = next(
+                            self.read(
+                                tags=base_tags + [TrendTag.task_best_fit_demodulation_matrices()],
+                                decoder=fits_array_decoder,
+                            )
+                        )
+
+                        logger.info(f"Smoothing demodulation matrices for {log_str}")
+                        smoothed_demod = self.smooth_demod_matrices(
+                            demod_matrices=raw_demod_matrices,
+                            fit_order=instrument_options.spatial_smoothing_fit_order,
+                            num_full_slit_pos=full_beam_shape[1],
+                        )
+
+                        # Reshaping the demodulation matrix to get rid of unit length dimensions
+                        logger.info(f"Resampling demodulation matrices for {log_str}")
+                        final_demod_matrices = self.reshape_demod_matrices(
+                            smoothed_demod, full_beam_shape
+                        )
+                        logger.info(
+                            f"Shape of resampled demodulation matrices: {final_demod_matrices.shape}"
+                        )
+
+                    with self.telemetry_span(f"Writing demodulation matrices for {log_str}"):
+                        self.write(
+                            data=final_demod_matrices,
+                            tags=base_tags + [TrendTag.task_processed_demodulation_matrices()],
+                            encoder=fits_array_encoder,
+                        )
+
+    def get_full_beam_shape(self, beam: Literal[1, 2]) -> tuple[int, int]:
+        """Read an INTERMEDIATE gain frame to measure the shape of a single beam array."""
+        gain_array = next(
+            self.read(
+                tags=[
+                    TrendTag.intermediate(),
+                    TrendTag.frame(),
+                    TrendTag.arm_id(self.arm_id),
+                    TrendTag.task_gain(),
+                ],
+                decoder=fits_array_decoder,
+            )
+        )
+        beam_array = extract_visp_beam(
+            gain_array, beam=beam, beam_border=self.parameters.visp_beam_border
+        )
+        return beam_array.shape
+
+    def smooth_demod_matrices(
+        self, demod_matrices: np.ndarray, fit_order: int, num_full_slit_pos: int
+    ) -> np.ndarray:
+        """
+        Smooth demodulation matrices in the spatial dimension.
+
+        The output will fully sample the spatial dimension so, as a side effect, this function also up-samples any data
+        that were binned spatially.
+
+        Smoothing is done using a RANSAC regression estimator to perform a polynomial fit with high resilience to outliers.
+
+        This method is taken directly from the ViSP L1 pipeline.
+        """
+        # We need to smooth the *modulation* (not DEmodulation) matrices to preserve the normalization of the Stokes-I
+        # values. linalg.pinv makes this easy
+        modulation_matrices = np.linalg.pinv(demod_matrices)
+
+        num_wave, num_binned_slit_pos, num_mod, num_stokes = modulation_matrices.shape
+
+        smoothed_mod = np.zeros((num_wave, num_full_slit_pos, num_mod, num_stokes))
+
+        # The binned abscissa is the "x" locations of the bins along the full spatial range
+        # The full abscissa is the "x" locations of all spatial pixels
+        # Add dummy dimensions because sklearn requires it
+        binned_abscissa = np.linspace(
+            start=0, stop=num_full_slit_pos, num=num_binned_slit_pos, endpoint=False
+        )[:, None]
+        full_abscissa = np.arange(num_full_slit_pos)[:, None]
+
+        model = self.build_RANSAC_model(fit_order=fit_order)
+        for w in range(num_wave):
+            for m in range(num_mod):
+                for s in range(num_stokes):
+                    curve = modulation_matrices[w, :, m, s]
+
+                    # Clean weirdo pixels
+                    fill_value = np.nanmedian(curve)
+                    curve[~np.isfinite(curve)] = fill_value
+
+                    model.fit(binned_abscissa, curve)
+                    fit_curve = model.predict(full_abscissa)
+                    smoothed_mod[w, :, m, s] = fit_curve
+
+        # Now compute the inverse again to return the DEmodulation matrices
+        smoothed_demod = np.linalg.pinv(smoothed_mod)
+
+        return smoothed_demod
+
+    def build_RANSAC_model(self, fit_order: int) -> Pipeline:
+        """
+        Build a scikit-learn pipeline from a set of estimators.
+
+        This method is taken directly from the ViSP L1 pipeline.
+        """
+        # PolynomialFeatures casts the pipeline as a polynomial fit
+        # see https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.PolynomialFeatures.html#sklearn.preprocessing.PolynomialFeatures
+        poly_feature = PolynomialFeatures(degree=fit_order)
+
+        # RobustScaler is a scale factor that is robust to outliers
+        # see https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.RobustScaler.html#sklearn.preprocessing.RobustScaler
+        scaler = RobustScaler()
+
+        # The RANSAC regressor iteratively sub-samples the input data and constructs a model with this sub-sample.
+        # The method used allows it to be robust to outliers.
+        # see https://scikit-learn.org/stable/modules/linear_model.html#ransac-regression
+        RANSAC = RANSACRegressor(
+            min_samples=self.parameters.visp_polcal_demod_spatial_smooth_min_samples
+        )
+
+        return make_pipeline(poly_feature, scaler, RANSAC)
+
+    def reshape_demod_matrices(
+        self, demod_matrices: np.ndarray, full_beam_shape: tuple[int, int]
+    ) -> np.ndarray:
+        """Upsample demodulation matrices to match the full beam size.
+
+        Given an input set of demodulation matrices with shape ``(X', Y', 4, M)`` resample the output to shape
+        ``(X, Y, 4, M)``, where ``X'`` and ``Y'`` are the binned size of the beam FOV, ``X`` and ``Y`` are the full beam shape, and ``M`` is the
+        number of modulator states.
+
+        If only a single demodulation matrix was made then it is returned as a single array with shape ``(4, M)``.
+
+
+        This method is taken directly from the ViSP L1 pipeline.
+
+        Parameters
+        ----------
+        demod_matrices
+            A set of demodulation matrices with shape ``(X', Y', 4, M)``
+
+        Returns
+        -------
+        If ``X'`` or ``Y'`` > 1 then upsampled matrices that are the full beam size ``(X, Y, 4, M)``. If ``X' == Y' == 1`` then a single matric for the whole FOV with shape ``(4, M)``
+
+        """
+        if len(demod_matrices.shape) != 4:
+            raise ValueError(
+                f"Expected demodulation matrices to have 4 dimensions. Got shape {demod_matrices.shape}"
+            )
+
+        # The non-demodulation matrix part of the larger array
+        data_shape = demod_matrices.shape[:2]
+        # The shape of a single demodulation matrix
+        demod_shape = demod_matrices.shape[-2:]
+        logger.info(f"Demodulation FOV sampling shape: {data_shape}")
+        logger.info(f"Demodulation matrix shape: {demod_shape}")
+        if data_shape == (1, 1):
+            # A single modulation matrix can be used directly, so just return it after removing extraneous dimensions
+            logger.info(f"Single demodulation matrix detected")
+            return demod_matrices[0, 0, :, :]
+
+        target_shape = full_beam_shape + demod_shape
+        logger.info(f"Target full-frame demodulation shape: {target_shape}")
+        full_frame_matrices = next(
+            resize_arrays(
+                demod_matrices, target_shape, order=self.parameters.visp_polcal_demod_upsample_order
+            )
+        )
+        return full_frame_matrices