dkist-processing-visp 2.20.14__py3-none-any.whl → 5.1.1__py3-none-any.whl

dkist_processing_visp/parsers/time.py

@@ -1,7 +1,9 @@
 """Stems for parsing constants and tags related to time header keys."""
-from collections import namedtuple
+
 from pathlib import Path
+from typing import NamedTuple
 
+from dkist_processing_common.models.fits_access import MetadataKey
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.flower_pot import Thorn
@@ -12,16 +14,12 @@ from dkist_processing_visp.models.constants import VispBudName
 from dkist_processing_visp.parsers.visp_l0_fits_access import VispL0FitsAccess
 
 
-class NonDarkTaskReadoutExpTimesBud(Stem):
-    """
-    Produce a tuple of all exposure times present in the dataset for ip task types that are not DARK.
-
-    POLCAL is included here because ViSP, unlike most other instruments, uses the IP task DARK frames for correction.
-    """
+class NonDarkNonPolcalTaskReadoutExpTimesBud(Stem):
+    """Produce a tuple of all exposure times present in the dataset for ip task types that are not DARK or POLCAL."""
 
     def __init__(self):
         super().__init__(stem_name=VispBudName.non_dark_task_readout_exp_times.value)
-        self.metadata_key = "sensor_readout_exposure_time_ms"
+        self.metadata_key = MetadataKey.sensor_readout_exposure_time_ms.name
 
     def setter(self, fits_obj: VispL0FitsAccess) -> float | SpilledDirt:
         """
@@ -36,7 +34,8 @@ class NonDarkTaskReadoutExpTimesBud(Stem):
         -------
         The exposure time associated with this fits object
         """
-        if fits_obj.ip_task_type.casefold() != TaskName.dark.value.casefold():
+        excluded_task_types = [TaskName.dark.value.casefold(), TaskName.polcal.value.casefold()]
+        if fits_obj.ip_task_type.casefold() not in excluded_task_types:
             raw_exposure_time = getattr(fits_obj, self.metadata_key)
             return round(raw_exposure_time, EXP_TIME_ROUND_DIGITS)
 
@@ -59,15 +58,23 @@ class NonDarkTaskReadoutExpTimesBud(Stem):
         return exposure_times
 
 
+class ReadoutExposureTimeContainer(NamedTuple):
+    """Named tuple to hold whether the task is dark and/or polcal along with the associated exposure time."""
+
+    is_dark: bool
+    is_polcal: bool
+    readout_exposure_time: float
+
+
 class DarkReadoutExpTimePickyBud(Stem):
     """Parse exposure times to ensure existence of the necessary DARK exposure times."""
 
-    ReadoutExposureTime = namedtuple("ReadoutExposureTime", ["is_dark", "readout_exposure_time"])
-    key_to_petal_dict: dict[str | Path, ReadoutExposureTime]  # For type hinting
+    ReadoutExposureTime: ReadoutExposureTimeContainer = ReadoutExposureTimeContainer
+    key_to_petal_dict: dict[str | Path, ReadoutExposureTimeContainer]  # For type hinting
 
     def __init__(self):
         super().__init__(stem_name=VispBudName.dark_readout_exp_time_picky_bud.value)
-        self.metadata_key = "sensor_readout_exposure_time_ms"
+        self.metadata_key = MetadataKey.sensor_readout_exposure_time_ms.name
 
     def setter(self, fits_obj: VispL0FitsAccess) -> tuple:
         """
@@ -84,8 +91,11 @@ class DarkReadoutExpTimePickyBud(Stem):
         raw_exposure_time = getattr(fits_obj, self.metadata_key)
         exposure_time = round(raw_exposure_time, EXP_TIME_ROUND_DIGITS)
         is_dark = fits_obj.ip_task_type.casefold() == TaskName.dark.value.casefold()
+        is_polcal = fits_obj.ip_task_type.casefold() == TaskName.polcal.value.casefold()
 
-        return self.ReadoutExposureTime(is_dark=is_dark, readout_exposure_time=exposure_time)
+        return self.ReadoutExposureTime(
+            is_dark=is_dark, is_polcal=is_polcal, readout_exposure_time=exposure_time
+        )
 
     def getter(self, key: str | Path) -> Thorn:
         """
@@ -109,7 +119,7 @@ class DarkReadoutExpTimePickyBud(Stem):
         required_readout_exp_times = {
             exp_time.readout_exposure_time
             for exp_time in readout_exp_tuples
-            if not exp_time.is_dark
+            if (not exp_time.is_dark and not exp_time.is_polcal)
         }
 
         required_exp_times_missing_from_dark_exposure_times = (

dkist_processing_visp/parsers/visp_l0_fits_access.py

@@ -1,7 +1,10 @@
 """ViSP FITS access for L0 data."""
+
 from astropy.io import fits
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 
+from dkist_processing_visp.models.fits_access import VispMetadataKey
+
 
 class VispL0FitsAccess(L0FitsAccess):
     """
@@ -29,11 +32,19 @@ class VispL0FitsAccess(L0FitsAccess):
     ):
         super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
 
-        self.number_of_modulator_states: int = self.header["VSPNUMST"]
-        self.raster_scan_step: int = self.header["VSPSTP"]
-        self.total_raster_steps: int = self.header["VSPNSTP"]
-        self.modulator_state: int = self.header["VSPSTNUM"]
-        self.polarimeter_mode: str = self.header["VISP_006"]
-        self.axis_1_type: str = self.header["CTYPE1"]
-        self.axis_2_type: str = self.header["CTYPE2"]
-        self.axis_3_type: str = self.header["CTYPE3"]
+        self.arm_id: int = self.header[VispMetadataKey.arm_id]
+        self.number_of_modulator_states: int = self.header[
+            VispMetadataKey.number_of_modulator_states
+        ]
+        self.raster_scan_step: int = self.header[VispMetadataKey.raster_scan_step]
+        self.total_raster_steps: int = self.header[VispMetadataKey.total_raster_steps]
+        self.modulator_state: int = self.header[VispMetadataKey.modulator_state]
+        self.polarimeter_mode: str = self.header[VispMetadataKey.polarimeter_mode]
+        self.grating_angle_deg: float = self.header[VispMetadataKey.grating_angle_deg]
+        self.arm_position_deg: float = self.header[VispMetadataKey.arm_position_deg]
+        self.grating_constant_inverse_mm: float = self.header[
+            VispMetadataKey.grating_constant_inverse_mm
+        ]
+        self.axis_1_type: str = self.header[VispMetadataKey.axis_1_type]
+        self.axis_2_type: str = self.header[VispMetadataKey.axis_2_type]
+        self.axis_3_type: str = self.header[VispMetadataKey.axis_3_type]

dkist_processing_visp/parsers/visp_l1_fits_access.py

@@ -1,4 +1,5 @@
 """ViSP FITS access for L1 data."""
+
 from astropy.io import fits
 from dkist_processing_common.parsers.l1_fits_access import L1FitsAccess
 

dkist_processing_visp/tasks/__init__.py

@@ -1,4 +1,5 @@
 """init."""
+
 from dkist_processing_visp.tasks.assemble_movie import *
 from dkist_processing_visp.tasks.background_light import *
 from dkist_processing_visp.tasks.dark import *

dkist_processing_visp/tasks/assemble_movie.py

@@ -1,4 +1,5 @@
 """ViSP-specific assemble movie task subclass."""
+
 from dkist_processing_common.tasks import AssembleMovie
 from PIL import ImageDraw
 

dkist_processing_visp/tasks/background_light.py

@@ -1,4 +1,5 @@
 """ViSP background light calibration task."""
+
 import gc
 import time
 
@@ -114,7 +115,7 @@ class BackgroundLightCalibration(
             del full_background_light
             gc.collect()
 
-        with self.apm_processing_step("Computing and logging quality metrics"):
+        with self.telemetry_span("Computing and logging quality metrics"):
             self.quality_store_task_type_counts(
                 task_type=VispTaskName.background.value,
                 total_frames=num_used_polcal_files,

dkist_processing_visp/tasks/dark.py

@@ -1,4 +1,5 @@
 """Visp dark task."""
+
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.fits import fits_array_encoder
 from dkist_processing_common.models.task_name import TaskName
@@ -49,7 +50,7 @@ class DarkCalibration(VispTaskBase, BeamAccessMixin, QualityMixin):
         required_readout_exp_times = list(self.constants.non_dark_task_readout_exp_times)
         logger.info(f"{required_readout_exp_times = }")
 
-        with self.apm_task_step(
+        with self.telemetry_span(
             f"Calculating dark frames for {self.constants.num_beams} beams and "
             f"{len(required_readout_exp_times)} readout exp times"
         ):
@@ -74,7 +75,7 @@ class DarkCalibration(VispTaskBase, BeamAccessMixin, QualityMixin):
                         fits_access_class=VispL0FitsAccess,
                     )
 
-                    with self.apm_processing_step(
+                    with self.telemetry_span(
                         f"Calculating dark for {readout_exp_time = } and {beam = }"
                     ):
                         readout_normalized_arrays = (
@@ -83,7 +84,7 @@ class DarkCalibration(VispTaskBase, BeamAccessMixin, QualityMixin):
                         )
                         averaged_dark_array = average_numpy_arrays(readout_normalized_arrays)
 
-                    with self.apm_writing_step(f"Writing dark for {readout_exp_time = } {beam = }"):
+                    with self.telemetry_span(f"Writing dark for {readout_exp_time = } {beam = }"):
                         self.write(
                             data=averaged_dark_array,
                             tags=VispTag.intermediate_frame_dark(
@@ -92,7 +93,7 @@ class DarkCalibration(VispTaskBase, BeamAccessMixin, QualityMixin):
                             encoder=fits_array_encoder,
                         )
 
-        with self.apm_processing_step("Computing and logging quality metrics"):
+        with self.telemetry_span("Computing and logging quality metrics"):
             no_of_raw_dark_frames: int = self.scratch.count_all(
                 tags=[
                     VispTag.input(),

dkist_processing_visp/tasks/geometric.py

@@ -3,12 +3,15 @@ Visp geometric calibration task.
 
 See :doc:`this page </geometric>` for more information.
 """
+
 from typing import Generator
 
 import numpy as np
+import peakutils
 import peakutils as pku
 import scipy.ndimage as spnd
 import scipy.optimize as spo
+import scipy.signal as sps
 import skimage.exposure as skie
 import skimage.metrics as skim
 import skimage.morphology as skimo
@@ -32,7 +35,6 @@ from dkist_processing_visp.models.tags import VispTag
 from dkist_processing_visp.parsers.visp_l0_fits_access import VispL0FitsAccess
 from dkist_processing_visp.tasks.mixin.beam_access import BeamAccessMixin
 from dkist_processing_visp.tasks.mixin.corrections import CorrectionsMixin
-from dkist_processing_visp.tasks.mixin.line_zones import LineZonesMixin
 from dkist_processing_visp.tasks.visp_base import VispTaskBase
 
 __all__ = ["GeometricCalibration"]
@@ -43,7 +45,6 @@ class GeometricCalibration(
     BeamAccessMixin,
     CorrectionsMixin,
     QualityMixin,
-    LineZonesMixin,
 ):
     """
     Task class for computing the spectral geometry. Geometry is represented by three quantities.
@@ -85,15 +86,15 @@ class GeometricCalibration(
         """
         # This lives outside the run() loops and has its own internal loops because the angle calculation
         # only happens for a single modstate
-        with self.apm_processing_step("Basic corrections"):
+        with self.telemetry_span("Basic corrections"):
             self.do_basic_corrections()
 
         for beam in range(1, self.constants.num_beams + 1):
-            with self.apm_task_step(f"Generating geometric calibrations for {beam = }"):
-                with self.apm_processing_step(f"Computing and writing angle for {beam = }"):
+            with self.telemetry_span(f"Generating geometric calibrations for {beam = }"):
+                with self.telemetry_span(f"Computing and writing angle for {beam = }"):
                     angle = self.compute_beam_angle(beam=beam)
                     if beam == 2:
-                        with self.apm_processing_step("Refining beam 2 angle"):
+                        with self.telemetry_span("Refining beam 2 angle"):
                             ang_corr = self.refine_beam2_angle(angle)
                         logger.info(f"Beam 2 angle refinement = {np.rad2deg(ang_corr)} deg")
                         angle += ang_corr
@@ -101,14 +102,12 @@ class GeometricCalibration(
                     self.write_angle(angle=angle, beam=beam)
 
                 for modstate in range(1, self.constants.num_modstates + 1):
-                    with self.apm_processing_step(
-                        f"Removing angle from {beam = } and {modstate = }"
-                    ):
+                    with self.telemetry_span(f"Removing angle from {beam = } and {modstate = }"):
                         angle_corr_array = self.remove_beam_angle(
                             angle=angle, beam=beam, modstate=modstate
                         )
 
-                    with self.apm_processing_step(
+                    with self.telemetry_span(
                         f"Computing state offset for {beam = } and {modstate = }"
                     ):
                         state_offset = self.compute_modstate_offset(
@@ -116,7 +115,7 @@ class GeometricCalibration(
                         )
                         self.write_state_offset(offset=state_offset, beam=beam, modstate=modstate)
 
-                    with self.apm_processing_step(
+                    with self.telemetry_span(
                         f"Removing state offset for {beam = } and {modstate = }"
                     ):
                         self.remove_state_offset(
@@ -126,13 +125,13 @@ class GeometricCalibration(
                             modstate=modstate,
                         )
 
-                with self.apm_processing_step(f"Computing spectral shifts for {beam = }"):
+                with self.telemetry_span(f"Computing spectral shifts for {beam = }"):
                     spec_shifts = self.compute_spectral_shifts(beam=beam)
 
-                with self.apm_writing_step(f"Writing spectral shifts for {beam = }"):
+                with self.telemetry_span(f"Writing spectral shifts for {beam = }"):
                     self.write_spectral_shifts(shifts=spec_shifts, beam=beam)
 
-        with self.apm_processing_step("Computing and logging quality metrics"):
+        with self.telemetry_span("Computing and logging quality metrics"):
             no_of_raw_geo_frames: int = self.scratch.count_all(
                 tags=[
                     VispTag.input(),
@@ -275,7 +274,12 @@ class GeometricCalibration(
         self.prep_input_solar_gain()
 
     def prep_lamp_gain(self):
-        """Apply dark corrections fo INPUT lamp frames."""
+        """
+        Create average, dark-corrected lamp gain images for each modstate from INPUT lamp gains.
+
+        This is different from the results of the `~dkist_processing_visp.tasks.lamp.LampCalibration` task because in
+        that task the hairlines are masked out, but here we *need* the hairlines to compute the rotation angle.
+        """
         for readout_exp_time in self.constants.lamp_readout_exp_times:
             for beam in range(1, self.constants.num_beams + 1):
                 logger.info(
@@ -1075,11 +1079,11 @@ class GeometricCalibration(
         `compute_single_state_offset`.
         """
         zone_kwargs = {
-            "prominence": self.parameters.solar_zone_prominence,
-            "width": self.parameters.solar_zone_width,
-            "bg_order": self.parameters.solar_zone_bg_order,
-            "normalization_percentile": self.parameters.solar_zone_normalization_percentile,
-            "rel_height": self.parameters.solar_zone_rel_height,
+            "prominence": self.parameters.geo_zone_prominence,
+            "width": self.parameters.geo_zone_width,
+            "bg_order": self.parameters.geo_zone_bg_order,
+            "normalization_percentile": self.parameters.geo_zone_normalization_percentile,
+            "rel_height": self.parameters.geo_zone_rel_height,
         }
         zones = self.compute_line_zones(array, **zone_kwargs)
         logger.info(f"Found {zones = }")
@@ -1089,6 +1093,114 @@ class GeometricCalibration(
 
         return mask
 
+    def compute_line_zones(
+        self,
+        spec_2d: np.ndarray,
+        prominence: float = 0.2,
+        width: float = 2,
+        bg_order: int = 22,
+        normalization_percentile: int = 99,
+        rel_height: float = 0.97,
+    ) -> list[tuple[int, int]]:
+        """
+        Identify spectral regions around strong spectra features.
+
+        Parameters
+        ----------
+        spec_2d
+            Data
+
+        prominence
+            Zone prominence threshold used to identify strong spectral features
+
+        width
+            Zone width
+
+        bg_order
+            Order of polynomial fit used to remove continuum when identifying strong spectral features
+
+        normalization_percentile
+            Compute this percentile of the data along a specified axis
+
+        rel_height
+            The relative height at which the peak width is measured as a percentage of its prominence. E.g., 1.0 measures
+            the peak width at the lowest contour line.
+
+        Returns
+        -------
+        regions
+            List of indices defining the found spectral lines
+
+        """
+        logger.info(
+            f"Finding zones using {prominence=}, {width=}, {bg_order=}, {normalization_percentile=}, and {rel_height=}"
+        )
+        # Compute average along slit to improve signal. Line smearing isn't important here
+        avg_1d = np.mean(spec_2d, axis=1)
+
+        # Convert to an emission spectrum and remove baseline continuum so peakutils has an easier time
+        em_spec = -1 * avg_1d + avg_1d.max()
+        em_spec /= np.nanpercentile(em_spec, normalization_percentile)
+        baseline = peakutils.baseline(em_spec, bg_order)
+        em_spec -= baseline
+
+        # Find indices of peaks
+        peak_idxs = sps.find_peaks(em_spec, prominence=prominence, width=width)[0]
+
+        # Find the rough width based only on the height of the peak
+        #  rips and lips are the right and left borders of the region around the peak
+        _, _, rips, lips = sps.peak_widths(em_spec, peak_idxs, rel_height=rel_height)
+
+        # Convert to ints so they can be used as indices
+        rips = np.floor(rips).astype(int)
+        lips = np.ceil(lips).astype(int)
+
+        # Remove any regions that are contained within another region
+        ranges_to_remove = self.identify_overlapping_zones(rips, lips)
+        rips = np.delete(rips, ranges_to_remove)
+        lips = np.delete(lips, ranges_to_remove)
+
+        return list(zip(rips, lips))
+
+    @staticmethod
+    def identify_overlapping_zones(rips: np.ndarray, lips: np.ndarray) -> list[int]:
+        """
+        Identify line zones that overlap with other zones. Any overlap greater than 1 pixel is flagged.
+
+        Parameters
+        ----------
+        rips
+            Right borders of the region around the peak
+
+        lips
+            Left borders of the region around the peak
+
+        Returns
+        -------
+        overlapping regions
+            List indices into the input arrays that represent an overlapped region that can be removed
+        """
+        all_ranges = [np.arange(zmin, zmax) for zmin, zmax in zip(rips, lips)]
+        ranges_to_remove = []
+        for i in range(len(all_ranges)):
+            target_range = all_ranges[i]
+            for j in range(i + 1, len(all_ranges)):
+                if (
+                    np.intersect1d(target_range, all_ranges[j]).size > 1
+                ):  # Allow for a single overlap just to be nice
+                    if target_range.size > all_ranges[j].size:
+                        ranges_to_remove.append(j)
+                        logger.info(
+                            f"Zone ({all_ranges[j][0]}, {all_ranges[j][-1]}) inside zone ({target_range[0]}, {target_range[-1]})"
+                        )
+                    else:
+                        ranges_to_remove.append(i)
+                        logger.info(
+                            f"Zone ({target_range[0]}, {target_range[-1]}) inside zone ({all_ranges[j][0]}, {all_ranges[j][-1]})"
+                        )
+
+        return ranges_to_remove
+
     @staticmethod
     def high_pass_filter_array(array: np.ndarray) -> np.ndarray:
         """Perform a simple high-pass filter to accentuate narrow features (hairlines and spectra)."""