dkist-processing-visp 2.20.14__py3-none-any.whl → 5.1.1__py3-none-any.whl

dkist_processing_visp/tests/header_models.py

@@ -8,6 +8,7 @@ values.
 This is very nice, but it's important to understand that the kwargs matter beyond just setting header values; they
 actually control the output of the generators themselves.
 """
+
 import uuid
 from random import choice
 from random import random
@@ -30,6 +31,12 @@ class VispHeaders(Spec122Dataset):
         num_modstates_header_value: int = 2,
         instrument: str = "visp",
         polarimeter_mode: str = "observe_polarimetric",
+        arm_id: int = 1,
+        ip_start_time: str = "2022-11-28T13:00:00",
+        ip_end_time: str = "2022-11-28T13:00:00",
+        grating_constant: float = 316.0,
+        grating_angle: float = -69.9,
+        arm_position: float = -4.0,
         **kwargs,
     ):
         super().__init__(
@@ -39,14 +46,32 @@ class VispHeaders(Spec122Dataset):
             instrument=instrument,
             **kwargs,
         )
+        self.add_constant_key("VISP_001", arm_id)
         self.add_constant_key("WAVELNTH", 656.30)
         self.add_constant_key("VISP_010", num_modstates_header_value)
         self.add_constant_key("ID___013", "TEST_PROPOSAL_ID")
         self.add_constant_key("VISP_006", polarimeter_mode)
         self.add_constant_key("PAC__005", "0")
         self.add_constant_key("PAC__007", "10")
+        self.add_constant_key("DKIST011", ip_start_time)
+        self.add_constant_key("DKIST012", ip_end_time)
         self.add_constant_key("FILE_ID", uuid.uuid4().hex)
+
+        self.add_constant_key("VISP_002", arm_position)
+        self.add_constant_key("VISP_013", grating_constant)
+        self.add_constant_key("VISP_015", grating_angle)
+
         self.num_modstates_header_value = num_modstates_header_value
+        self.add_constant_key("CAM__001", "camera_id")
+        self.add_constant_key("CAM__002", "camera_name")
+        self.add_constant_key("CAM__003", 1)
+        self.add_constant_key("CAM__009", 1)
+        self.add_constant_key("CAM__010", 1)
+        self.add_constant_key("CAM__011", 1)
+        self.add_constant_key("CAM__012", 1)
+        self.add_constant_key("ID___014", "v1")
+        self.add_constant_key("TELTRACK", "Fixed Solar Rotation Tracking")
+        self.add_constant_key("TTBLTRCK", "fixed angle on sun")
 
     @key_function("VISP_011")
     def current_modstate(self, key: str):
@@ -72,6 +97,8 @@ class VispHeadersInputDarkFrames(VispHeaders):
         num_modstates: int,
         exp_time: float = 1.0,
         readout_exp_time: float = 2.0,
+        ip_start_time="2022-11-28T13:44:00",
+        ip_end_time="2022-11-28T13:45:00",
         **kwargs,
     ):
         ################################################
@@ -84,13 +111,13 @@ class VispHeadersInputDarkFrames(VispHeaders):
             array_shape,
             time_delta,
             num_modstates_header_value=num_modstates,
+            ip_start_time=ip_start_time,
+            ip_end_time=ip_end_time,
             **kwargs,
         )
         self.add_constant_key("DKIST004", TaskName.dark.value.lower())
         self.add_constant_key("DKIST008", 1)
         self.add_constant_key("DKIST009", 1)
-        self.add_constant_key("DKIST011", "2022-11-28T13:44:00")
-        self.add_constant_key("DKIST012", "2022-11-28T13:45:00")
         self.add_constant_key("VISP_019", 1)  # Num raster steps
         self.add_constant_key("VISP_020", 1)  # Current raster step
         self.add_constant_key("ID___004")
@@ -100,6 +127,12 @@ class VispHeadersInputDarkFrames(VispHeaders):
         self.add_constant_key("CAM__004", exp_time)
         self.add_constant_key("CAM__005", readout_exp_time)
         self.add_constant_key("CAM__014", 10)  # Num frames per FPA
+        self.add_constant_key("TELSCAN", "Raster")
+        self.add_constant_key("PAC__002", "lamp")
+        self.add_constant_key("PAC__003", "on")
+        self.add_constant_key("PAC__004", "Sapphire Polarizer")
+        self.add_constant_key("PAC__006", "SiO2 OC")
+        self.add_constant_key("PAC__008", "FieldStop (5arcmin)")
 
     @key_function("VISP_011")
     def current_modstate(self, key: str) -> int:
@@ -114,6 +147,8 @@ class VispHeadersInputLampGainFrames(VispHeaders):
         num_modstates: int,
         exp_time: float = 10.0,
         readout_exp_time: float = 20.0,
+        ip_start_time="2022-11-28T13:46:00",
+        ip_end_time="2022-11-28T13:47:00",
         **kwargs,
     ):
         ################################################
@@ -126,14 +161,14 @@ class VispHeadersInputLampGainFrames(VispHeaders):
             array_shape,
             time_delta,
             num_modstates_header_value=num_modstates,
+            ip_start_time=ip_start_time,
+            ip_end_time=ip_end_time,
             **kwargs,
         )
         self.add_constant_key("DKIST004", TaskName.gain.value.lower())
         self.add_constant_key("PAC__002", "lamp")
         self.add_constant_key("DKIST008", 1)
         self.add_constant_key("DKIST009", 1)
-        self.add_constant_key("DKIST011", "2022-11-28T13:46:00")
-        self.add_constant_key("DKIST012", "2022-11-28T13:47:00")
         self.add_constant_key("VISP_019", 1)
         self.add_constant_key("VISP_020", 1)
         self.add_constant_key("PAC__003", "on")
@@ -155,6 +190,8 @@ class VispHeadersInputSolarGainFrames(VispHeaders):
         num_modstates: int,
         exp_time: float = 20.0,
         readout_exp_time: float = 40.0,
+        ip_start_time="2022-11-28T13:48:00",
+        ip_end_time="2022-11-28T13:49:00",
         **kwargs,
     ):
         ################################################
@@ -167,16 +204,20 @@ class VispHeadersInputSolarGainFrames(VispHeaders):
             array_shape,
             time_delta,
             num_modstates_header_value=num_modstates,
+            ip_start_time=ip_start_time,
+            ip_end_time=ip_end_time,
             **kwargs,
         )
         self.add_constant_key("DKIST004", TaskName.gain.value.lower())
         self.add_constant_key("DKIST008", 1)
         self.add_constant_key("DKIST009", 1)
-        self.add_constant_key("DKIST011", "2022-11-28T13:48:00")
-        self.add_constant_key("DKIST012", "2022-11-28T13:49:00")
         self.add_constant_key("VISP_019", 1)
         self.add_constant_key("VISP_020", 1)
         self.add_constant_key("PAC__002", "clear")
+        self.add_constant_key("PAC__003", "off")
+        self.add_constant_key("PAC__004", "clear")
+        self.add_constant_key("PAC__006", "clear")
+        self.add_constant_key("PAC__008", "FieldStop (5arcmin)")
         self.add_constant_key("TELSCAN", "Raster")
         self.add_constant_key("ID___004")
         self.add_constant_key("CAM__004", exp_time)
@@ -197,6 +238,8 @@ class VispHeadersInputPolcalFrames(VispHeaders):
         num_cs_steps: int = 1,
         exp_time: float = 0.01,
         readout_exp_time: float = 0.02,
+        ip_start_time="2022-11-28T13:50:00",
+        ip_end_time="2022-11-28T13:51:00",
         **kwargs,
     ):
         ################################################
@@ -209,31 +252,36 @@ class VispHeadersInputPolcalFrames(VispHeaders):
             array_shape,
             time_delta,
             num_modstates_header_value=num_modstates,
+            ip_start_time=ip_start_time,
+            ip_end_time=ip_end_time,
             **kwargs,
         )
         self.index_to_modstate = list(range(1, num_modstates + 1)) * num_cs_steps
         self.index_to_cs_step = sum([[c] * num_modstates for c in range(num_cs_steps)], [])
         self.polarizer_choices = ["Sapphire Polarizer", "clear"]
-        self.retarder_choices = ["SiO2 SAR", "clear"]
+        self.retarder_choices = ["SiO2 OC", "clear"]
 
         self.add_constant_key("DKIST004", TaskName.polcal.value.lower())
         self.add_constant_key("DKIST008", 1)
         self.add_constant_key("DKIST009", 1)
-        self.add_constant_key("DKIST011", "2022-11-28T13:50:00")
-        self.add_constant_key("DKIST012", "2022-11-28T13:51:00")
         self.add_constant_key("VISP_019", 1)
         self.add_constant_key("VISP_020", 1)
         self.add_constant_key("TELSCAN", "Raster")
         self.add_constant_key("ID___004")
         self.add_constant_key("PAC__004", "Sapphire Polarizer")
         self.add_constant_key("PAC__005", "60.")
-        self.add_constant_key("PAC__006", "clear")
         self.add_constant_key("PAC__007", "0.0")
         self.add_constant_key("PAC__008", "FieldStop (5arcmin)")
         self.add_constant_key("CAM__004", exp_time)
         self.add_constant_key("CAM__005", readout_exp_time)
         self.add_constant_key("CAM__014", 10)
 
+    @key_function("PAC__006")
+    def retarder_name(self, key: str) -> str:
+        if self.index % 2:
+            return self.retarder_choices[1]
+        return self.retarder_choices[0]
+
     @property
     def current_cs_step(self) -> int:
         # There is no header value for CS step; this property is used to help data generator fixtures with tagging
@@ -251,15 +299,89 @@ class VispHeadersInputPolcalFrames(VispHeaders):
     def pol_angle(self, key: str) -> float:
         return random() * 120.0
 
-    @key_function("PAC__006")
-    def retarder(self, key: str) -> str:
-        return choice(self.retarder_choices)
-
     @key_function("PAC__007")
     def ret_angle(self, key: str) -> float:
         return random() * 120.0
 
 
+class VispHeadersInputPolcalDarkFrames(VispHeaders):
+    def __init__(
+        self,
+        array_shape: tuple[int, ...],
+        time_delta: float,
+        num_modstates: int,
+        exp_time: float = 0.01,
+        readout_exp_time: float = 0.02,
+        ip_start_time="2022-11-28T13:50:00",
+        ip_end_time="2022-11-28T13:51:00",
+        **kwargs,
+    ):
+        num_frames = num_modstates
+        dataset_shape = (num_frames, *array_shape[-2:])
+        super().__init__(
+            dataset_shape,
+            array_shape,
+            time_delta,
+            ip_start_time=ip_start_time,
+            ip_end_time=ip_end_time,
+            **kwargs,
+        )
+        self.add_constant_key("DKIST004", TaskName.polcal.value.lower())
+        self.add_constant_key("DKIST008", 1)
+        self.add_constant_key("DKIST009", 1)
+        self.add_constant_key("VISP_019", 1)
+        self.add_constant_key("VISP_020", 1)
+        self.add_constant_key("TELSCAN", "Raster")
+        self.add_constant_key("ID___004")
+        self.add_constant_key("PAC__004", "clear")
+        self.add_constant_key("PAC__005", "60.")
+        self.add_constant_key("PAC__006", "clear")
+        self.add_constant_key("PAC__007", "0.0")
+        self.add_constant_key("PAC__008", "DarkShutter")
+        self.add_constant_key("CAM__004", exp_time)
+        self.add_constant_key("CAM__005", readout_exp_time)
+        self.add_constant_key("CAM__014", 10)
+
+
+class VispHeadersInputPolcalGainFrames(VispHeaders):
+    def __init__(
+        self,
+        array_shape: tuple[int, ...],
+        time_delta: float,
+        num_modstates: int,
+        exp_time: float = 0.01,
+        readout_exp_time: float = 0.02,
+        ip_start_time="2022-11-28T13:50:00",
+        ip_end_time="2022-11-28T13:51:00",
+        **kwargs,
+    ):
+        num_frames = num_modstates
+        dataset_shape = (num_frames, *array_shape[-2:])
+        super().__init__(
+            dataset_shape,
+            array_shape,
+            time_delta,
+            ip_start_time=ip_start_time,
+            ip_end_time=ip_end_time,
+            **kwargs,
+        )
+        self.add_constant_key("DKIST004", TaskName.polcal.value.lower())
+        self.add_constant_key("DKIST008", 1)
+        self.add_constant_key("DKIST009", 1)
+        self.add_constant_key("VISP_019", 1)
+        self.add_constant_key("VISP_020", 1)
+        self.add_constant_key("TELSCAN", "Raster")
+        self.add_constant_key("ID___004")
+        self.add_constant_key("PAC__004", "clear")
+        self.add_constant_key("PAC__005", "60.")
+        self.add_constant_key("PAC__006", "clear")
+        self.add_constant_key("PAC__007", "0.0")
+        self.add_constant_key("PAC__008", "FieldStop (5arcmin)")
+        self.add_constant_key("CAM__004", exp_time)
+        self.add_constant_key("CAM__005", readout_exp_time)
+        self.add_constant_key("CAM__014", 10)
+
+
 class VispHeadersValidObserveFrames(VispHeaders):
     def __init__(
         self,
@@ -271,6 +393,9 @@ class VispHeadersValidObserveFrames(VispHeaders):
         exp_time: float = 15.0,
         readout_exp_time: float = 30.0,
         abort_last_step: bool = False,
+        ip_start_time="2022-11-28T13:55:00",
+        ip_end_time="2022-11-28T13:56:00",
+        grating_constant=316,
         **kwargs,
     ):
         ################################################
@@ -284,7 +409,15 @@ class VispHeadersValidObserveFrames(VispHeaders):
             num_frames -= num_dropped_frames
 
         dataset_shape = (num_frames, *array_shape[-2:])
-        super().__init__(dataset_shape, array_shape, time_delta, **kwargs)
+        super().__init__(
+            dataset_shape,
+            array_shape,
+            time_delta,
+            ip_start_time=ip_start_time,
+            ip_end_time=ip_end_time,
+            grating_constant=grating_constant,
+            **kwargs,
+        )
 
         self.index_to_map = sum(
             [[map_num + 1] * num_modstates * num_raster_steps for map_num in range(num_maps)], []
@@ -305,8 +438,6 @@ class VispHeadersValidObserveFrames(VispHeaders):
 
         self.num_raster_steps = num_raster_steps
         self.add_constant_key("DKIST004", TaskName.observe.value.lower())
-        self.add_constant_key("DKIST011", "2022-11-28T13:55:00")  # IP start time
-        self.add_constant_key("DKIST012", "2022-11-28T13:56:00")  # IP end time
         self.add_constant_key("ID___004")
         self.add_constant_key("WAVELNTH", 656.28)
         self.add_constant_key("EXPER_ID", "EXPERIMENT ID")
@@ -353,6 +484,8 @@ class VispHeadersValidCalibratedFrames(VispHeaders):
         exp_time: float = 15.0,
         readout_exp_time: float = 30.0,
         wcs_axis_names: tuple[str, str] | None = None,
+        ip_start_time="2022-11-28T13:55:00",
+        ip_end_time="2022-11-28T13:56:00",
         **kwargs,
     ):
         ################################################
@@ -366,7 +499,13 @@ class VispHeadersValidCalibratedFrames(VispHeaders):
         num_frames = num_maps * num_raster_steps * num_stokes
         dataset_shape = (num_frames, *array_shape[-2:])
         super().__init__(
-            dataset_shape, array_shape, time_delta, polarimeter_mode=polarimeter_mode, **kwargs
+            dataset_shape,
+            array_shape,
+            time_delta,
+            polarimeter_mode=polarimeter_mode,
+            ip_start_time=ip_start_time,
+            ip_end_time=ip_end_time,
+            **kwargs,
         )
 
         stokes_list = ["I", "Q", "U", "V"][:num_stokes]
@@ -380,8 +519,6 @@ class VispHeadersValidCalibratedFrames(VispHeaders):
 
         self.num_raster_steps = num_raster_steps
         self.add_constant_key("DKIST004", TaskName.observe.value.lower())
-        self.add_constant_key("DKIST011", "2022-11-28T13:55:00")  # IP start time
-        self.add_constant_key("DKIST012", "2022-11-28T13:56:00")  # IP end time
         self.add_constant_key("ID___004")
         self.add_constant_key("WAVELNTH", 656.28)
         self.add_constant_key("EXPER_ID", "EXPERIMENT ID")

dkist_processing_visp/tests/local_trial_workflows/l0_cals_only.py

@@ -4,12 +4,11 @@ import os
 import sys
 from dataclasses import asdict
 from datetime import datetime
-from pathlib import Path
 from random import randint
 
-from astropy.io import fits
-from dkist_header_validator import spec122_validator
+from dkist_processing_common.codecs.basemodel import basemodel_encoder
 from dkist_processing_common.manual import ManualProcessing
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.tasks import WorkflowTaskBase
 from dkist_service_configuration.logging import logger
 
@@ -38,15 +37,19 @@ from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers impor
 from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveInstPolCal
 from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveLampCal
 from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SaveSolarCal
-from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
-    set_observe_wavelength_task,
-)
 from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SetNumModstates
 from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SetObserveExpTime
 from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
     SetObserveIpStartTime,
 )
 from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import SetPolarimeterMode
+from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
+    set_observe_wavelength_task,
+)
+from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import tag_inputs_task
+from dkist_processing_visp.tests.local_trial_workflows.local_trial_helpers import (
+    translate_122_to_214l0_task,
+)
 
 
 class DatetimeEncoder(json.JSONEncoder):
@@ -61,43 +64,19 @@ class DatetimeEncoder(json.JSONEncoder):
         return super().default(obj)
 
 
-class Translate122To214L0(WorkflowTaskBase):
-    def run(self) -> None:
-        raw_dir = Path(self.scratch.scratch_base_path) / f"VISP{self.recipe_run_id:03n}"
-        if not os.path.exists(self.scratch.workflow_base_path):
-            os.makedirs(self.scratch.workflow_base_path)
-
-        if not raw_dir.exists():
-            raise FileNotFoundError(
-                f"Expected to find a raw VISP{{run_id:03n}} folder in {self.scratch.scratch_base_path}"
-            )
-
-        for file in raw_dir.glob("*.FITS"):
-            translated_file_name = Path(self.scratch.workflow_base_path) / os.path.basename(file)
-            logger.info(f"Translating {file} -> {translated_file_name}")
-            hdl = fits.open(file)
-
-            header = spec122_validator.validate_and_translate_to_214_l0(
-                hdl[0].header, return_type=fits.HDUList
-            )[0].header
-
-            comp_hdu = fits.CompImageHDU(header=header, data=hdl[0].data)
-            comp_hdl = fits.HDUList([fits.PrimaryHDU(), comp_hdu])
-            comp_hdl.writeto(translated_file_name, overwrite=True)
-
-            hdl.close()
-            del hdl
-            comp_hdl.close()
-            del comp_hdl
-
-
 class CreateInputDatasetParameterDocument(WorkflowTaskBase):
     def run(self) -> None:
-        doc_path = self.scratch.workflow_base_path / "input_dataset_parameters.json"
-        with open(doc_path, "w") as f:
-            f.write(json.dumps(self.input_dataset_document_simple_parameters_part))
-        self.tag(doc_path, VispTag.input_dataset_parameters())
-        logger.info(f"Wrote input dataset doc to {doc_path}")
+        relative_path = "input_dataset_parameters.json"
+        self.write(
+            data=InputDatasetPartDocumentList(
+                doc_list=self.input_dataset_document_simple_parameters_part
+            ),
+            relative_path=relative_path,
+            tags=VispTag.input_dataset_parameters(),
+            encoder=basemodel_encoder,
+            overwrite=True,
+        )
+        logger.info(f"Wrote input dataset parameter doc to {relative_path}")
 
     @property
     def input_dataset_document_simple_parameters_part(self):
@@ -121,37 +100,6 @@ class CreateInputDatasetParameterDocument(WorkflowTaskBase):
         return parameters_list
 
 
-def tag_inputs_task(suffix: str):
-    class TagInputs(WorkflowTaskBase):
-        def run(self) -> None:
-            logger.info(f"Looking in {os.path.abspath(self.scratch.workflow_base_path)}")
-            input_file_list = list(self.scratch.workflow_base_path.glob(f"*.{suffix}"))
-            if len(input_file_list) == 0:
-                raise FileNotFoundError(
-                    f"Did not find any files matching '*.{suffix}' in {self.scratch.workflow_base_path}"
-                )
-            for file in input_file_list:
-                logger.info(f"Found {file}")
-                self.tag(path=file, tags=[VispTag.input(), VispTag.frame()])
-
-    return TagInputs
-
-
-def setup_APM_config() -> None:
-    mesh_config = {
-        "system-monitoring-log-apm": {
-            "mesh_address": "system-monitoring-log-apm.service.sim.consul",
-            "mesh_port": 8200,
-        },
-        "automated-processing-scratch-inventory": {"mesh_address": "localhost", "mesh_port": 6379},
-        "internal-api-gateway": {"mesh_address": "localhost", "mesh_port": 80},
-    }
-    apm_options = {"TRANSACTION_MAX_SPANS": 10000}
-    os.environ["MESH_CONFIG"] = json.dumps(mesh_config)
-    os.environ["ELASTIC_APM_ENABLED"] = "true"
-    os.environ["ELASTIC_APM_OTHER_OPTIONS"] = json.dumps(apm_options)
-
-
 def main(
     scratch_path: str,
     suffix: str = "FITS",
@@ -165,11 +113,8 @@ def main(
     load_geometric: bool = False,
     load_solar: bool = False,
     load_inst_pol: bool = False,
-    use_apm: bool = False,
     dummy_wavelength: float = 630.0,
 ):
-    if use_apm:
-        setup_APM_config()
     with ManualProcessing(
         workflow_path=scratch_path,
         recipe_run_id=recipe_run_id,
@@ -178,7 +123,7 @@ def main(
         workflow_version="GROGU",
     ) as manual_processing_run:
         if not skip_translation:
-            manual_processing_run.run_task(task=Translate122To214L0)
+            manual_processing_run.run_task(task=translate_122_to_214l0_task(suffix=suffix))
         if only_translate:
             return
         manual_processing_run.run_task(task=CreateInputDatasetParameterDocument)
@@ -302,7 +247,6 @@ if __name__ == "__main__":
         help="Load instrument polarization calibration from previously saved run",
         action="store_true",
     )
-    parser.add_argument("-A", "--use-apm", help="Send APM spans to SIM", action="store_true")
     args = parser.parse_args()
     sys.exit(
         main(
@@ -318,6 +262,5 @@ if __name__ == "__main__":
             load_geometric=args.load_geometric,
             load_solar=args.load_solar,
             load_inst_pol=args.load_inst_pol,
-            use_apm=args.use_apm,
         )
     )