dkist-processing-visp 5.1.2rc1__py3-none-any.whl → 5.2.0__py3-none-any.whl

dkist_processing_visp/models/parameters.py

@@ -5,6 +5,7 @@ from random import randint
 from typing import Any
 
 import astropy.units as u
+from astropy.units import Quantity
 from dkist_processing_common.models.parameters import ParameterArmIdMixin
 from dkist_processing_common.models.parameters import ParameterBase
 from dkist_processing_common.models.parameters import ParameterWavelengthMixin
@@ -200,7 +201,7 @@ class VispParameters(ParameterBase, ParameterWavelengthMixin, ParameterArmIdMixi
     @property
     def solar_vignette_dispersion_bounds_fraction(self) -> float:
         """
-        Define the ± fraction away from the initial value for bounds on dispersion when fitting the initial vignette signal.
+        Define the ± fraction from the initial value for bounds on dispersion when fitting the initial vignette signal.
 
         This value should be between 0 and 1. For example, the minimum bound is `init_value * (1 - solar_vignette_dispersion_bounds_fraction)`.
         """
@@ -230,6 +231,33 @@ class VispParameters(ParameterBase, ParameterWavelengthMixin, ParameterArmIdMixi
         """Return fractional number of samples required for the RANSAC regressor used to fit the 2D vignette signal."""
         return self._find_most_recent_past_value("visp_solar_vignette_min_samples")
 
+    @property
+    def wavecal_crval_bounds_px(self) -> Quantity:
+        """
+        Define the bounds (in *pix*) on crval when performing wavecal fitting.
+
+        The actual bounds on the value of crval are equal to ± the initial dispersion times this number. Note that the
+        total range searched by the fitting algorithm will be twice this number (in pixels).
+        """
+        return self._find_most_recent_past_value("visp_wavecal_crval_bounds_px") * u.pix
+
+    @property
+    def wavecal_dispersion_bounds_fraction(self) -> Quantity:
+        """
+        Define the ± fraction from the initial value for bounds on dispersion when performing wavecal fitting.
+
+        This value should be between 0 and 1. For example, the minimum bound is `init_value * (1 - wavecal_dispersion_bounds_fraction)`.
+        """
+        return self._find_most_recent_past_value("visp_wavecal_dispersion_bounds_fraction")
+
+    @property
+    def wavecal_incident_light_angle_bounds_deg(self) -> Quantity:
+        """Define the bounds (in *deg*) on incident_light_angle when performing wavecal fitting."""
+        return (
+            self._find_most_recent_past_value("visp_wavecal_incident_light_angle_bounds_deg")
+            * u.deg
+        )
+
     @property
     def wavecal_camera_lens_parameters(self) -> list[u.Quantity]:
         r"""
@@ -279,6 +307,20 @@ class VispParameters(ParameterBase, ParameterWavelengthMixin, ParameterArmIdMixi
         """Define the initial guess for opacity factor in wavecal fits."""
         return self._find_most_recent_past_value("visp_wavecal_init_opacity_factor")
 
+    @property
+    def wavecal_fit_kwargs(self) -> dict[str, Any]:
+        """Define extra keyword arguments to pass to the wavelength calibration fitter."""
+        doc_dict = self._find_most_recent_past_value("visp_wavecal_fit_kwargs")
+        rng_kwarg = dict()
+        fitting_method = doc_dict.get("method", False)
+        if fitting_method in ["basinhopping", "differential_evolution", "dual_annealing"]:
+            rng = randint(1, 1_000_000)
+            rng_kwarg["rng"] = rng
+
+        # The order here allows us to override `rng` in a parameter value
+        fit_kwargs = rng_kwarg | doc_dict
+        return fit_kwargs
+
     @property
     def polcal_spatial_median_filter_width_px(self) -> int:
         """Return the size of the median filter to apply in the spatial dimension to polcal data."""

dkist_processing_visp/models/tags.py

@@ -2,6 +2,7 @@
 
 from enum import Enum
 
+from dkist_processing_common.models.tags import StemName
 from dkist_processing_common.models.tags import Tag
 
 from dkist_processing_visp.models.task_name import VispTaskName
@@ -63,6 +64,16 @@ class VispTag(Tag):
         """
         return cls.format_tag(VispStemName.map_scan, map_scan_num)
 
+    @classmethod
+    def task_characteristic_spectra(cls) -> str:
+        """Tags intermediate characteristic spectra."""
+        return cls.format_tag(StemName.task, VispTaskName.solar_char_spec.value)
+
+    @classmethod
+    def task_wavelength_calibration(cls) -> str:
+        """Tags wavelength calibration."""
+        return cls.format_tag(StemName.task, VispTaskName.wavelength_calibration.value)
+
     ##################
     # Composite tags #
     ##################

dkist_processing_visp/models/task_name.py

@@ -7,3 +7,5 @@ class VispTaskName(str, Enum):
     """Controlled list of task tag names."""
 
     background = "BACKGROUND"
+    solar_char_spec = "SOLAR_CHAR_SPEC"
+    wavelength_calibration = "WAVELENGTH_CALIBRATION"

dkist_processing_visp/tasks/geometric.py

@@ -1086,7 +1086,7 @@ class GeometricCalibration(
             "rel_height": self.parameters.geo_zone_rel_height,
         }
         zones = self.compute_line_zones(array, **zone_kwargs)
-        logger.info(f"Found {zones = }")
+        logger.info(f"Found zones = {[tuple(int(i) for i in z) for z in zones]}")
         mask = np.zeros(array.shape).astype(bool)
         for z in zones:
             mask[z[0] : z[1], :] = True

dkist_processing_visp/tasks/science.py

@@ -12,6 +12,7 @@ from astropy.time import TimeDelta
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.fits import fits_array_decoder
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
+from dkist_processing_common.codecs.json import json_decoder
 from dkist_processing_common.models.fits_access import MetadataKey
 from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_common.tasks.mixin.quality import QualityMixin
@@ -42,6 +43,7 @@ class CalibrationCollection:
     angle: dict
     state_offset: dict
     spec_shift: dict
+    wavelength_calibration_header: dict
     demod_matrices: dict | None
 
     @cached_property
@@ -176,6 +178,15 @@ class ScienceCalibration(
         spec_shift_dict = dict()
         demod_dict = dict() if self.constants.correct_for_polarization else None
 
+        # WaveCal
+        #########
+        wavecal_header = next(
+            self.read(
+                tags=[VispTag.intermediate(), VispTag.task_wavelength_calibration()],
+                decoder=json_decoder,
+            )
+        )
+
         for beam in range(1, self.constants.num_beams + 1):
             for readout_exp_time in self.constants.observe_readout_exp_times:
                 # Dark
@@ -270,6 +281,7 @@ class ScienceCalibration(
             angle=angle_dict,
             state_offset=state_offset_dict,
             spec_shift=spec_shift_dict,
+            wavelength_calibration_header=wavecal_header,
             demod_matrices=demod_dict,
         )
 
@@ -761,22 +773,31 @@ class ScienceCalibration(
         """
         Update calibrated headers with any information gleaned during science calibration.
 
-        Right now all this does is put map scan values in the header.
+        #. Apply the wavelength calibration header values
+        #. Add map scan keywords
+        #. Adjust CRPIX values based on any chopping needed to return only regions where the beams overlap
 
         Parameters
         ----------
-        header : fits.Header
+        header
             The header to update
 
-        map_scan : int
+        map_scan
             Current map scan
 
+        calibrations
+            Container of intermediate calibration objects. Used to figure out how much to adjust CRPIX values.
+
         Returns
         -------
         fits.Header
             Updated header
 
         """
+        # Apply the wavelength calibration
+        # This needs to be done prior to adjusting CRPIX values below
+        header.update(calibrations.wavelength_calibration_header)
+
         # Update the map scan number
         header["VSPNMAPS"] = self.constants.num_map_scans
         header["VSPMAP"] = map_scan
@@ -784,27 +805,51 @@ class ScienceCalibration(
         # Adjust the CRPIX values if the beam overlap slicing chopped from the start of the array
         x_slice, y_slice = calibrations.beams_overlap_slice
 
+        # Note: We KNOW that `x_slice` and `y_slice` correspond to the *array* dimensions corresponding to spectral and
+        # spatial directions, respectively. Some early ViSP data swap the *WCS* dimensions so that CRPIX1 contains
+        # information about the spectral axis, even though the spectral axis is always in the 0th array axis (and thus
+        # the second WCS axis because FITS and numpy are backwards; are we confused yet?).
+        #
+        # We want the adjustment of the CRPIX values to produce accurate WCS information, even if they're associated
+        # with the wrong array axes. For this reason we dynamically associate the WCS axis number with `x_slice` and
+        # `y_slice` via the `*_wcs_axis_num` properties.
+        #
+        # To say it differently, we KNOW that `x_slice` always refers to chopping in the spectral dimension, so we need
+        # to update the CRPIX associate with the spectral WCS, no matter which WCS axis that is.
+
         # This if catches 0's and Nones
         if x_slice.start:
             # .start will only be non-None or 0 if the slice is from the start. In this case we need to update the WCS
             logger.info(
-                f"Adjusting CRPIX2 from {header['CRPIX2']} to {header['CRPIX2'] - x_slice.start}"
+                f"Adjusting spectral CRPIX{self.spectral_wcs_axis_num} from "
+                f"{header[f'CRPIX{self.spectral_wcs_axis_num}']} to {header[f'CRPIX{self.spectral_wcs_axis_num}'] - x_slice.start}"
+            )
+            header[f"CRPIX{self.spectral_wcs_axis_num}"] = (
+                header[f"CRPIX{self.spectral_wcs_axis_num}"] - x_slice.start
             )
-            header["CRPIX2"] = header["CRPIX2"] - x_slice.start
             logger.info(
-                f"Adjusting CRPIX2A from {header['CRPIX2A']} to {header['CRPIX2A'] - x_slice.start}"
+                f"Adjusting spectral CRPIX{self.spectral_wcs_axis_num}A from "
+                f"{header[f'CRPIX{self.spectral_wcs_axis_num}A']} to {header[f'CRPIX{self.spectral_wcs_axis_num}A'] - x_slice.start}"
+            )
+            header[f"CRPIX{self.spectral_wcs_axis_num}A"] = (
+                header[f"CRPIX{self.spectral_wcs_axis_num}A"] - x_slice.start
             )
-            header["CRPIX2A"] = header["CRPIX2A"] - x_slice.start
 
         if y_slice.start:
             logger.info(
-                f"Adjusting CRPIX1 from {header['CRPIX1']} to {header['CRPIX1'] - y_slice.start}"
+                f"Adjusting spatial CRPIX{self.spatial_wcs_axis_num} from "
+                f"{header[f'CRPIX{self.spatial_wcs_axis_num}']} to {header[f'CRPIX{self.spatial_wcs_axis_num}'] - y_slice.start}"
+            )
+            header[f"CRPIX{self.spatial_wcs_axis_num}"] = (
+                header[f"CRPIX{self.spatial_wcs_axis_num}"] - y_slice.start
             )
-            header["CRPIX1"] = header["CRPIX1"] - y_slice.start
             logger.info(
-                f"Adjusting CRPIX1A from {header['CRPIX1A']} to {header['CRPIX1A'] - y_slice.start}"
+                f"Adjusting spatial CRPIX{self.spatial_wcs_axis_num}A from "
+                f"{header[f'CRPIX{self.spatial_wcs_axis_num}A']} to {header[f'CRPIX{self.spatial_wcs_axis_num}A'] - y_slice.start}"
+            )
+            header[f"CRPIX{self.spatial_wcs_axis_num}A"] = (
+                header[f"CRPIX{self.spatial_wcs_axis_num}A"] - y_slice.start
             )
-            header["CRPIX1A"] = header["CRPIX1A"] - y_slice.start
 
         return header
 
@@ -900,3 +945,35 @@ class ScienceCalibration(
 
         filename = next(self.read(tags=tags))
         logger.info(f"Wrote intermediate file for {tags = } to {filename}")
+
+    @cached_property
+    def spectral_wcs_axis_num(self) -> int:
+        """
+        Return the WCS axis number corresponding to wavelength.
+
+        We need to check this dynamically because some early ViSP data got the WCS backward w.r.t. the array dimensions.
+        """
+        try:
+            spectral_axis_num = next(
+                (i for i in range(1, 4) if getattr(self.constants, f"axis_{i}_type") == "AWAV")
+            )
+        except StopIteration as e:
+            raise ValueError("Could not find WCS axis with type AWAV") from e
+
+        return spectral_axis_num
+
+    @cached_property
+    def spatial_wcs_axis_num(self) -> int:
+        """
+        Return the WCS axis number corresponding to Helioprojective latitude.
+
+        We need to check this dynamically because some early ViSP data got the WCS backward w.r.t. the array dimensions.
+        """
+        try:
+            spatial_axis_num = next(
+                (i for i in range(1, 4) if getattr(self.constants, f"axis_{i}_type") == "HPLT-TAN")
+            )
+        except StopIteration as e:
+            raise ValueError("Cound not find WCS axis with type HPLT-TAN") from e
+
+        return spatial_axis_num

dkist_processing_visp/tasks/solar.py

@@ -8,14 +8,11 @@ from typing import Callable
 import astropy.units as u
 import numpy as np
 import scipy.ndimage as spnd
-from astropy.time import Time
 from astropy.units import Quantity
-from astropy.wcs import WCS
 from dkist_processing_common.codecs.asdf import asdf_encoder
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.fits import fits_array_decoder
 from dkist_processing_common.codecs.fits import fits_array_encoder
-from dkist_processing_common.models.dkist_location import location_of_dkist
 from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_common.tasks.mixin.quality import QualityMixin
 from dkist_processing_math.arithmetic import divide_arrays_by_array
@@ -38,9 +35,7 @@ from solar_wavelength_calibration import UnitlessBoundRange
 from solar_wavelength_calibration import WavelengthCalibrationFitter
 from solar_wavelength_calibration import WavelengthCalibrationParameters
 from solar_wavelength_calibration.fitter.wavelength_fitter import FitResult
-from solar_wavelength_calibration.fitter.wavelength_fitter import WavelengthParameters
 from solar_wavelength_calibration.fitter.wavelength_fitter import calculate_initial_crval_guess
-from sunpy.coordinates import HeliocentricInertial
 
 from dkist_processing_visp.models.metric_code import VispMetricCode
 from dkist_processing_visp.models.tags import VispTag
@@ -48,15 +43,15 @@ from dkist_processing_visp.parsers.visp_l0_fits_access import VispL0FitsAccess
 from dkist_processing_visp.tasks.mixin.beam_access import BeamAccessMixin
 from dkist_processing_visp.tasks.mixin.corrections import CorrectionsMixin
 from dkist_processing_visp.tasks.visp_base import VispTaskBase
+from dkist_processing_visp.tasks.wavelength_calibration import compute_initial_dispersion
+from dkist_processing_visp.tasks.wavelength_calibration import compute_input_wavelength_vector
+from dkist_processing_visp.tasks.wavelength_calibration import compute_order
+from dkist_processing_visp.tasks.wavelength_calibration import get_doppler_velocity
 
 __all__ = [
     "SolarCalibration",
     "WavelengthCalibrationParametersWithContinuum",
     "polynomial_continuum_model",
-    "compute_order",
-    "compute_initial_dispersion",
-    "compute_doppler_velocity",
-    "compute_input_wavelength_vector",
 ]
 
 
@@ -128,7 +123,7 @@ class SolarCalibration(
     QualityMixin,
 ):
     """
-    Task class for generating Solar Gain images for each beam/modstate.
+    Task class for generating Solar Gain images for each beam.
 
     Parameters
     ----------
@@ -190,8 +185,8 @@ class SolarCalibration(
                     pixel_pitch=self.parameters.wavecal_pixel_pitch_micron_per_pix,
                 )
 
-                doppler_velocity = compute_doppler_velocity(
-                    time_of_observation=self.constants.solar_gain_ip_start_time
+                doppler_velocity = get_doppler_velocity(
+                    solar_gain_ip_start_time=self.constants.solar_gain_ip_start_time
                 )
 
                 self._log_wavecal_parameters(
@@ -247,8 +242,10 @@ class SolarCalibration(
                 self.write(
                     data=char_spec,
                     encoder=fits_array_encoder,
-                    tags=[VispTag.debug(), VispTag.frame()],
-                    relative_path=f"DEBUG_SC_CHAR_SPEC_BEAM_{beam}.dat",
+                    tags=[
+                        VispTag.intermediate_frame(beam=beam),
+                        VispTag.task_characteristic_spectra(),
+                    ],
                     overwrite=True,
                 )
 
@@ -830,9 +827,6 @@ class SolarCalibration(
         beam
             The beam number for this array
 
-        modstate
-            The modulator state for this array
-
 
         Returns
         -------
@@ -958,7 +952,7 @@ class SolarCalibration(
         self, dispersion: Quantity, order: int, doppler_velocity: Quantity
     ) -> None:
         """Log initial guess and instrument-derived wavecal parameters."""
-        logger.info(f"central wavelength = {self.constants.wavelength * u.nm !s}")
+        logger.info(f"central_wavelength = {self.constants.wavelength * u.nm !s}")
         logger.info(f"{dispersion = !s}")
         logger.info(f"{order = }")
         logger.info(f"grating constant = {self.constants.grating_constant_inverse_mm !s}")
@@ -996,190 +990,3 @@ def polynomial_continuum_model(
     """
     coeffs = [fit_parameters[f"poly_coeff_{i:02n}"].value for i in range(fit_order + 1)]
     return np.polyval(coeffs, abscissa)
-
-
-##################################################################################
-# TODO: The following definitions should go in the wavecal module once it exists #
-##################################################################################
-
-
-def compute_order(
-    central_wavelength: Quantity,
-    incident_light_angle: Quantity,
-    reflected_light_angle: Quantity,
-    grating_constant: Quantity,
-) -> int:
-    r"""
-    Compute the spectral order from the spectrograph setup.
-
-    From the grating equation, the spectral order, :math:`m`:, is
-
-    .. math::
-        m = \frac{\sin\alpha + \sin\beta}{G \lambda}
-
-    where :math:`\alpha` and :math:`\beta` are the incident and reflected light angles, respectively, :math:`G` is the
-    grating constant (lines per mm), and :math:`\lambda` is the central wavelength. All of these values come from the
-    input headers.
-
-    Parameters
-    ----------
-    central_wavelength
-        Wavelength of the center of the spectral window.
-
-    incident_light_angle
-        Angle of light incident to the spectrograph grating. Often called :math:`\alpha`.
-
-    reflected_light_angle
-        Angle of light reflected from spectrograph grating. Often called :math:`\beta`.
-
-    grating_constant
-        Grating constant of the spectrograph grating [lines per mm]
-
-    Returns
-    -------
-    spectral_order
-        The order of the given spectrograph configuration
-    """
-    return int(
-        (np.sin(incident_light_angle) + np.sin(reflected_light_angle))
-        / (grating_constant * central_wavelength)
-    )
-
-
-def compute_initial_dispersion(
-    central_wavelength: Quantity,
-    incident_light_angle: Quantity,
-    reflected_light_angle: Quantity,
-    lens_parameters: list[Quantity],
-    pixel_pitch: Quantity,
-) -> Quantity:
-    r"""
-    Compute the dispersion (:math:`d\,\lambda/d\, px`) given the spectrograph setup.
-
-    The dispersion is given via
-
-    .. math::
-        d\,\lambda / d\, px = \frac{p \lambda_0 \cos\beta}{f (\sin\alpha + \sin\beta)}
-
-    where :math:`p` is the pixel pitch (microns per pix), :math:`\lambda_0` is the central wavelength, :math:`f` is the
-    camera focal length, and :math:`\alpha` and :math:`\beta` are the incident and reflected light angles, respectively.
-    :math:`\lambda_0`, :math:`\alpha`, and :math:`\beta` are taken from input headers, while :math:`f` and :math:`p` are
-    pipeline parameters.
-
-    Parameters
-    ----------
-    central_wavelength
-        Wavelength of the center of the spectral window.
-
-    incident_light_angle
-        Angle of light incident to the spectrograph grating. Often called :math:`\alpha`.
-
-    reflected_light_angle
-        Angle of light reflected from spectrograph grating. Often called :math:`\beta`.
-
-    lens_parameters
-        Parameterization of lense focal length as zero, first, and second orders of wavelength. If the total focal
-        length of the lens is :math:`f = a_0 + a_1\lambda + a_2\lambda^2` then this list is :math:`[a_0, a_1, a_2]`.
-
-    pixel_pitch
-        The physical size of a single pixel
-
-    Returns
-    -------
-    dispersion
-       The computed dispersion in units of nm / px
-    """
-    camera_focal_length = lens_parameters[0] + central_wavelength * (
-        lens_parameters[1] + central_wavelength * lens_parameters[2]
-    )
-    logger.info(f"{camera_focal_length = !s}")
-
-    linear_dispersion = (
-        camera_focal_length
-        * (np.sin(incident_light_angle) + np.sin(reflected_light_angle))
-        / (np.cos(reflected_light_angle) * central_wavelength)
-    )
-
-    dispersion = pixel_pitch / linear_dispersion
-
-    return dispersion.to(u.nm / u.pix)
-
-
-def compute_doppler_velocity(time_of_observation: str) -> Quantity:
-    """Find the speed at which DKIST is moving relative to the Sun's center.
-
-    Positive values refer to when DKIST is moving away from the sun.
-
-    Parameters
-    ----------
-    time_of_observation
-        Time at which to compute the relative Dopper velocity. Any string that can be parsed by `astropy.time.Time` is
-        acceptable.
-
-    Returns
-    -------
-    doppler_velocity
-        The relative velocity between observer and the Sun with units of km / s
-    """
-    coord = location_of_dkist.get_gcrs(obstime=Time(time_of_observation))
-    heliocentric_coord = coord.transform_to(HeliocentricInertial(obstime=Time(time_of_observation)))
-    obs_vr_kms = heliocentric_coord.d_distance
-    return obs_vr_kms
-
-
-def compute_input_wavelength_vector(
-    *,
-    central_wavelength: Quantity,
-    dispersion: Quantity,
-    grating_constant: Quantity,
-    order: int,
-    incident_light_angle: Quantity,
-    num_spec_px: int,
-) -> u.Quantity:
-    r"""
-    Compute a wavelength vector based on information about the spectrograph setup.
-
-    The parameterization of the grating equation is via `astropy.wcs.WCS`, which follows section 5 of
-    `Greisen et al (2006) <https://ui.adsabs.harvard.edu/abs/2006A%26A...446..747G/abstract>`_.
-
-    Parameters
-    ----------
-    central_wavelength
-        Wavelength at the center of the spectral window. This function forces the value of the output vector to be
-        ``central_wavelength`` at index ``num_spec // 2 + 1``.
-
-    dispersion
-        Spectrograph dispersion [nm / px]
-
-    grating_constant
-        Grating constant of the spectrograph grating [lines per mm]
-
-    order
-        Spectrograph order
-
-    incident_light_angle
-        Angle of light incident to the spectrograph grating. Often called :math:`\alpha`.
-
-    num_spec_px
-        The length of the output wavelength vector. Defines size and physical limits of the output.
-
-    Returns
-    -------
-    wave_vec
-        1D array of length ``num_spec`` containing the wavelength values described by the input WCS parameterization.
-        The units of this array will be nanometers.
-    """
-    wavelength_parameters = WavelengthParameters(
-        crpix=num_spec_px // 2 + 1,
-        crval=central_wavelength.to_value(u.nm),
-        dispersion=dispersion.to_value(u.nm / u.pix),
-        grating_constant=grating_constant.to_value(1 / u.mm),
-        order=order,
-        incident_light_angle=incident_light_angle.to_value(u.deg),
-        cunit="nm",
-    )
-    header = wavelength_parameters.to_header(axis_num=1)
-    wcs = WCS(header)
-    input_wavelength_vector = wcs.spectral.pixel_to_world(np.arange(num_spec_px)).to(u.nm)
-
-    return input_wavelength_vector