dkist-processing-visp 2.20.14__py3-none-any.whl → 5.1.1__py3-none-any.whl

dkist_processing_visp/tasks/visp_base.py

@@ -1,14 +1,14 @@
 """ViSP base class."""
+
 from abc import ABC
 
 from dkist_processing_common.tasks import WorkflowTaskBase
-from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetMixin
 
 from dkist_processing_visp.models.constants import VispConstants
 from dkist_processing_visp.models.parameters import VispParameters
 
 
-class VispTaskBase(WorkflowTaskBase, InputDatasetMixin, ABC):
+class VispTaskBase(WorkflowTaskBase, ABC):
     """
     Task class for base ViSP tasks.
 
@@ -42,7 +42,8 @@ class VispTaskBase(WorkflowTaskBase, InputDatasetMixin, ABC):
             workflow_version=workflow_version,
         )
         self.parameters = VispParameters(
-            self.input_dataset_parameters,
+            scratch=self.scratch,
             obs_ip_start_time=self.constants.obs_ip_start_time,
             wavelength=self.constants.wavelength,
+            arm_id=self.constants.arm_id,
         )

dkist_processing_visp/tasks/write_l1.py

@@ -1,14 +1,17 @@
 """Visp write L1 task."""
+
 from functools import cache
 from typing import Literal
 
 import astropy.units as u
 from astropy.io import fits
+from dkist_processing_common.models.fits_access import MetadataKey
 from dkist_processing_common.tasks import WriteL1Frame
 from dkist_processing_common.tasks.write_l1 import WavelengthRange
 from dkist_service_configuration.logging import logger
 
 from dkist_processing_visp.models.constants import VispConstants
+from dkist_processing_visp.models.fits_access import VispMetadataKey
 
 cached_info_logger = cache(logger.info)
 __all__ = ["VispWriteL1Frame"]
@@ -54,7 +57,7 @@ class VispWriteL1Frame(WriteL1Frame):
         """
         # Correct the headers for the number of map and scan steps per map due to potential observation aborts
         header["VSPNMAPS"] = self.constants.num_map_scans
-        header["VSPNSTP"] = self.constants.num_raster_steps
+        header[VispMetadataKey.total_raster_steps] = self.constants.num_raster_steps
 
         if stokes.upper() not in self.constants.stokes_params:
             raise ValueError("The stokes parameter must be one of I, Q, U, V")
@@ -71,24 +74,27 @@ class VispWriteL1Frame(WriteL1Frame):
                 header[f"DTYPE{i}"] = "SPATIAL"
                 header[f"DPNAME{i}"] = "spatial along slit"
                 header[f"DWNAME{i}"] = "helioprojective latitude"
-                header[f"CNAME{i}"] = "helioprojective latitude"
+                if i < header["NAXIS"]:
+                    header[f"CNAME{i}"] = "helioprojective latitude"
                 header[f"DUNIT{i}"] = header[f"CUNIT{i}"]
             elif axis_type == "AWAV":
                 header[f"DNAXIS{i}"] = header[f"NAXIS{i}"]
                 header[f"DTYPE{i}"] = "SPECTRAL"
                 header[f"DPNAME{i}"] = "dispersion axis"
                 header[f"DWNAME{i}"] = "wavelength"
-                header[f"CNAME{i}"] = "wavelength"
+                if i < header["NAXIS"]:
+                    header[f"CNAME{i}"] = "wavelength"
                 header[f"DUNIT{i}"] = header[f"CUNIT{i}"]
             elif axis_type == "HPLN-TAN":
                 header[f"DNAXIS{i}"] = self.constants.num_raster_steps
                 header[f"DTYPE{i}"] = "SPATIAL"
                 header[f"DPNAME{i}"] = "raster scan step number"
                 header[f"DWNAME{i}"] = "helioprojective longitude"
-                header[f"CNAME{i}"] = "helioprojective longitude"
+                if i < header["NAXIS"]:
+                    header[f"CNAME{i}"] = "helioprojective longitude"
                 header[f"DUNIT{i}"] = header[f"CUNIT{i}"]
                 # Current position in raster scan which counts from zero
-                header[f"DINDEX{i}"] = header["VSPSTP"] + 1
+                header[f"DINDEX{i}"] = header[VispMetadataKey.raster_scan_step] + 1
             else:
                 raise ValueError(
                     f"Unexpected axis type. Expected ['HPLT-TAN', 'AWAV', 'HPLN-TAN']. Got {axis_type}"
@@ -101,13 +107,14 @@ class VispWriteL1Frame(WriteL1Frame):
         if self.constants.num_map_scans > 1:
             cached_info_logger("Adding map scan dataset axis")
             num_axis += 1
-            header[
-                f"DNAXIS{num_axis}"
-            ] = self.constants.num_map_scans  # total number of raster scans in the dataset
+            header[f"DNAXIS{num_axis}"] = (
+                self.constants.num_map_scans
+            )  # total number of raster scans in the dataset
             header[f"DTYPE{num_axis}"] = "TEMPORAL"
             header[f"DPNAME{num_axis}"] = "raster map repeat number"
             header[f"DWNAME{num_axis}"] = "time"
-            header[f"CNAME{num_axis}"] = "time"
+            if num_axis < header["NAXIS"]:
+                header[f"CNAME{num_axis}"] = "time"
             header[f"DUNIT{num_axis}"] = "s"
             # Temporal position in dataset
             header[f"DINDEX{num_axis}"] = header["VSPMAP"]  # Current raster scan
@@ -120,7 +127,8 @@ class VispWriteL1Frame(WriteL1Frame):
             header[f"DTYPE{num_axis}"] = "STOKES"
             header[f"DPNAME{num_axis}"] = "polarization state"
             header[f"DWNAME{num_axis}"] = "polarization state"
-            header[f"CNAME{num_axis}"] = "polarization state"
+            if num_axis < header["NAXIS"]:
+                header[f"CNAME{num_axis}"] = "polarization state"
             header[f"DUNIT{num_axis}"] = ""
             # Stokes position in dataset - stokes axis goes from 1-4
             header[f"DINDEX{num_axis}"] = self.constants.stokes_params.index(stokes.upper()) + 1
@@ -140,6 +148,9 @@ class VispWriteL1Frame(WriteL1Frame):
         header["NBIN3"] = 1
         header["NBIN"] = header["NBIN1"] * header["NBIN2"] * header["NBIN3"]
 
+        # Values don't have any units because they are relative to disk center
+        header["BUNIT"] = ("", "Values are relative to disk center. See calibration docs.")
+
         return header
 
     def calculate_date_end(self, header: fits.Header) -> str:
@@ -185,3 +196,20 @@ class VispWriteL1Frame(WriteL1Frame):
             min=u.Quantity(minimum, unit=wavelength_unit),
             max=u.Quantity(maximum, unit=wavelength_unit),
         )
+
+    def add_timing_headers(self, header: fits.Header) -> fits.Header:
+        """
+        Add timing headers to the FITS header.
+
+        This method adds or updates headers related to frame timings.
+        """
+        # The source data is based on L0 data but L1 data takes L0 cadence * modstates to obtain.
+        # This causes both the cadence and the full exposure time to be num_modstates times longer.
+        header["CADENCE"] = self.constants.average_cadence * self.constants.num_modstates
+        header["CADMIN"] = self.constants.minimum_cadence * self.constants.num_modstates
+        header["CADMAX"] = self.constants.maximum_cadence * self.constants.num_modstates
+        header["CADVAR"] = self.constants.variance_cadence * self.constants.num_modstates
+        header[MetadataKey.fpa_exposure_time_ms] = (
+            header[MetadataKey.fpa_exposure_time_ms] * self.constants.num_modstates
+        )
+        return header

dkist_processing_visp/tests/conftest.py

@@ -1,8 +1,10 @@
 import json
 from dataclasses import asdict
 from dataclasses import dataclass
+from dataclasses import field
 from dataclasses import is_dataclass
 from random import randint
+from typing import Any
 from typing import Callable
 from typing import Type
 
@@ -12,10 +14,15 @@ from astropy.io import fits
 from dkist_data_simulator.spec122 import Spec122Dataset
 from dkist_header_validator.translator import sanitize_to_spec214_level1
 from dkist_header_validator.translator import translate_spec122_to_spec214_l0
+from dkist_processing_common.codecs.basemodel import basemodel_encoder
 from dkist_processing_common.codecs.fits import fits_array_encoder
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.tasks import WorkflowTaskBase
 
+# Don't remove this; tests will break
+from dkist_processing_common.tests.mock_metadata_store import fake_gql_client
+
 from dkist_processing_visp.models.constants import VispConstants
 from dkist_processing_visp.models.parameters import VispParameters
 from dkist_processing_visp.models.tags import VispTag
@@ -38,6 +45,7 @@ def init_visp_constants_db():
 
 @dataclass
 class VispConstantsDb:
+    ARM_ID: int = 1
     POLARIMETER_MODE: str = "observe_polarimetric"
     OBS_IP_START_TIME: str = "2022-11-28T13:54:00"
     NUM_MODSTATES: int = 10
@@ -61,8 +69,12 @@ class VispConstantsDb:
     LAMP_READOUT_EXP_TIMES: tuple[float] = (200.0,)
     SOLAR_READOUT_EXP_TIMES: tuple[float] = (2.0,)
     OBSERVE_READOUT_EXP_TIMES: tuple[float] = (0.02,)
-    POLCAL_READOUT_EXP_TIMES: tuple[float] = ()
+    POLCAL_READOUT_EXP_TIMES: tuple[float] = (0.02,)
     SPECTRAL_LINE: str = "VISP Ca II H"
+    INCIDENT_LIGHT_ANGLE_DEG: float = 73.22
+    REFLECTED_LIGHT_ANGLE_DEG: float = 64.92
+    GRATING_CONSTANT_INVERSE_MM: float = 316.0
+    SOLAR_GAIN_IP_START_TIME: str = "2025-09-24T20:00:00"
     STOKES_PARAMS: tuple[str] = (
         "I",
         "Q",
@@ -81,6 +93,7 @@ class VispConstantsDb:
     AXIS_1_TYPE: str = "HPLT-TAN"
     AXIS_2_TYPE: str = "AWAV"
     AXIS_3_TYPE: str = "HPLN-TAN"
+    RETARDER_NAME: str = "SiO2 OC"
 
 
 @pytest.fixture()
@@ -124,19 +137,59 @@ class VispInputDatasetParameterValues:
     visp_geo_upsample_factor: float = 10.0
     visp_geo_max_shift: float = 40.0
     visp_geo_poly_fit_order: int = 3
-    visp_solar_spectral_avg_window: WavelengthParameter = WavelengthParameter(
-        values=(800, 800, 800, 800)
+    visp_geo_zone_prominence: WavelengthParameter = WavelengthParameter(values=(0.2, 0.2, 0.3, 0.2))
+    visp_geo_zone_width: WavelengthParameter = WavelengthParameter(values=(7, 2, 3, 2))
+    visp_geo_zone_bg_order: WavelengthParameter = WavelengthParameter(values=(21, 22, 11, 22))
+    visp_geo_zone_normalization_percentile: WavelengthParameter = WavelengthParameter(
+        values=(90, 99, 90, 90)
+    )
+    visp_geo_zone_rel_height: float = 0.97
+    visp_solar_spatial_median_filter_width_px: WavelengthParameter = WavelengthParameter(
+        values=(250, 250, 250, 250)
     )
     visp_solar_characteristic_spatial_normalization_percentile: float = 90.0
-    visp_solar_zone_prominence: WavelengthParameter = WavelengthParameter(
-        values=(0.2, 0.2, 0.3, 0.2)
+    visp_solar_vignette_initial_continuum_poly_fit_order: int = 6
+    visp_solar_vignette_crval_bounds_px: float = 7
+    visp_solar_vignette_dispersion_bounds_fraction: float = 0.02
+    visp_solar_vignette_wavecal_fit_kwargs: dict[str, Any] = field(
+        default_factory=lambda: {
+            "method": "differential_evolution",
+            "init": "halton",
+            "popsize": 1,
+            "tol": 1e-10,
+        }
     )
-    visp_solar_zone_width: WavelengthParameter = WavelengthParameter(values=(7, 2, 3, 2))
-    visp_solar_zone_bg_order: WavelengthParameter = WavelengthParameter(values=(21, 22, 11, 22))
-    visp_solar_zone_normalization_percentile: WavelengthParameter = WavelengthParameter(
-        values=(90, 99, 90, 90)
+    visp_solar_vignette_spectral_poly_fit_order: int = 12
+    visp_solar_vignette_min_samples: float = 0.9
+    visp_wavecal_camera_lens_parameters_1: tuple[float, float, float] = (
+        0.7613,
+        1.720e-4,
+        -8.139e-8,
+    )
+    visp_wavecal_camera_lens_parameters_2: tuple[float, float, float] = (
+        0.9512,
+        2.141e-4,
+        -1.014e-7,
+    )
+    visp_wavecal_camera_lens_parameters_3: tuple[float, float, float] = (
+        0.1153e1,
+        2.595e-4,
+        -1.230e-7,
+    )
+    visp_wavecal_pixel_pitch_micron_per_pix: float = 6.5
+    visp_wavecal_atlas_download_config: dict[str, str] = field(
+        default_factory=lambda: {
+            "base_url": "doi:10.5281/zenodo.14646787/",
+            "telluric_reference_atlas_file_name": "telluric_reference_atlas.npy",
+            "telluric_reference_atlas_hash_id": "md5:8db5e12508b293bca3495d81a0747447",
+            "solar_reference_atlas_file_name": "solar_reference_atlas.npy",
+            "solar_reference_atlas_hash_id": "md5:84ab4c50689ef235fe5ed4f7ee905ca0",
+        }
     )
-    visp_solar_zone_rel_height: float = 0.97
+    visp_wavecal_init_crval_guess_normalization_percentile: float = 95
+    visp_wavecal_init_resolving_power: int = 150000
+    visp_wavecal_init_straylight_fraction: float = 0.2
+    visp_wavecal_init_opacity_factor: float = 5.0
     visp_polcal_spatial_median_filter_width_px: int = 10
     visp_polcal_num_spatial_bins: int = 10
     visp_polcal_demod_spatial_smooth_fit_order: int = 17
@@ -144,7 +197,6 @@ class VispInputDatasetParameterValues:
     visp_polcal_demod_upsample_order: int = 3
     visp_pac_remove_linear_I_trend: bool = True
     visp_pac_fit_mode: str = "use_M12_I_sys_per_step"
-    visp_pac_init_set: str = "OCCal_VIS"
 
 
 @pytest.fixture(scope="session")
@@ -157,11 +209,37 @@ def testing_obs_ip_start_time() -> str:
     return "1946-11-20T12:34:56"
 
 
+@pytest.fixture(scope="session")
+def testing_grating_constant() -> float:
+    # Just make it different than the defaults in header_models.py
+    return 317.2
+
+
+@pytest.fixture(scope="session")
+def testing_grating_angle() -> float:
+    return -43.2
+
+
+@pytest.fixture(scope="session")
+def testing_arm_position() -> float:
+    return -5.3
+
+
+@pytest.fixture(scope="session")
+def testing_solar_ip_start_time() -> str:
+    return "1946-11-21T12:34:56"
+
+
+@pytest.fixture(scope="session")
+def testing_arm_id() -> int:
+    return 2
+
+
 @pytest.fixture(scope="session")
 def input_dataset_document_simple_parameters_part():
     """Convert a dataclass of parameterValues into an actual input dataset parameters part."""
 
-    def get_input_dataset_parameters_part(parameter_values: dataclass):
+    def make_input_dataset_parameters_part(parameter_values: dataclass):
         parameters_list = []
         value_id = randint(1000, 2000)
         for pn, pv in asdict(parameter_values).items():
@@ -178,35 +256,42 @@ def input_dataset_document_simple_parameters_part():
             parameters_list.append(parameter)
         return parameters_list
 
-    return get_input_dataset_parameters_part
+    return make_input_dataset_parameters_part
 
 
 @pytest.fixture(scope="session")
 def assign_input_dataset_doc_to_task(
-    input_dataset_document_simple_parameters_part, testing_obs_ip_start_time, testing_wavelength
+    input_dataset_document_simple_parameters_part,
+    testing_obs_ip_start_time,
+    testing_wavelength,
+    testing_arm_id,
 ):
     def update_task(
-        task,
+        task: WorkflowTaskBase,
         parameter_values,
         parameter_class=VispParameters,
         obs_ip_start_time=testing_obs_ip_start_time,
+        arm_id=testing_arm_id,
     ):
-        doc_path = task.scratch.workflow_base_path / "dataset_parameters.json"
-        with open(doc_path, "w") as f:
-            f.write(json.dumps(input_dataset_document_simple_parameters_part(parameter_values)))
-        task.tag(doc_path, VispTag.input_dataset_parameters())
+        task.write(
+            data=InputDatasetPartDocumentList(
+                doc_list=input_dataset_document_simple_parameters_part(parameter_values)
+            ),
+            tags=VispTag.input_dataset_parameters(),
+            encoder=basemodel_encoder,
+        )
         task.parameters = parameter_class(
-            task.input_dataset_parameters,
+            scratch=task.scratch,
             obs_ip_start_time=obs_ip_start_time,
             wavelength=testing_wavelength,
+            arm_id=str(arm_id),
         )
 
     return update_task
 
 
 def make_random_data(frame: Spec122Dataset) -> np.ndarray:
-    shape = frame.array_shape[1:]
-    data = np.random.random(shape)
+    data = np.random.random(frame.array_shape)
 
     return data
 
@@ -338,28 +423,14 @@ def write_intermediate_background_to_task(
         )
 
 
-def write_intermediate_lamp_to_task(
-    task, *, lamp_signal: float, beam: int, modstate: int, data_shape: tuple[int, int]
-):
-    lamp_array = np.ones(data_shape) * lamp_signal
-    task.write(
-        data=lamp_array,
-        tags=[
-            VispTag.intermediate_frame(beam=beam, modstate=modstate),
-            VispTag.task_lamp_gain(),
-        ],
-        encoder=fits_array_encoder,
-    )
-
-
 def write_intermediate_solar_to_task(
-    task, *, solar_signal: float, beam: int, modstate: int, data_shape: tuple[int, int]
+    task, *, solar_signal: float, beam: int, data_shape: tuple[int, int]
 ):
     solar_array = np.ones(data_shape) * solar_signal
     task.write(
         data=solar_array,
         tags=[
-            VispTag.intermediate_frame(beam=beam, modstate=modstate),
+            VispTag.intermediate_frame(beam=beam),
             VispTag.task_solar_gain(),
         ],
         encoder=fits_array_encoder,
@@ -409,19 +480,46 @@ def write_intermediate_geometric_to_task(
 def write_dummy_intermediate_solar_cals_to_task(
     task,
     *,
-    num_modstates: int,
     data_shape: tuple[int, int],
 ):
     solar_signal = 1.0
     for beam in [1, 2]:
-        for modstate in range(1, num_modstates + 1):
-            write_intermediate_solar_to_task(
-                task=task,
-                solar_signal=solar_signal,
-                beam=beam,
-                modstate=modstate,
-                data_shape=data_shape,
-            )
+        write_intermediate_solar_to_task(
+            task=task,
+            solar_signal=solar_signal,
+            beam=beam,
+            data_shape=data_shape,
+        )
+
+
+def write_intermediate_polcal_darks_to_task(
+    task, *, dark_signal: float, readout_exp_time: float, data_shape: tuple[int, int]
+):
+    dark_cal = np.ones(data_shape) * dark_signal
+    # Need a dark for each beam
+    for beam in [1, 2]:
+        task.write(
+            data=dark_cal,
+            tags=VispTag.intermediate_frame_polcal_dark(
+                beam=beam, readout_exp_time=readout_exp_time
+            ),
+            encoder=fits_array_encoder,
+        )
+
+
+def write_intermediate_polcal_gains_to_task(
+    task, *, gain_signal: float, readout_exp_time: float, data_shape: tuple[int, int]
+):
+    gain_cal = np.ones(data_shape) * gain_signal
+    # Need a dark for each beam
+    for beam in [1, 2]:
+        task.write(
+            data=gain_cal,
+            tags=VispTag.intermediate_frame_polcal_gain(
+                beam=beam, readout_exp_time=readout_exp_time
+            ),
+            encoder=fits_array_encoder,
+        )
 
 
 def tag_on_map_raster_stokes(frame: VispHeadersValidCalibratedFrames) -> list[str]: