dkist-processing-dlnirsp 0.32.8__py3-none-any.whl → 0.33.0__py3-none-any.whl

dkist_processing_dlnirsp/tests/local_trial_workflows/l0_solar_gain_as_science.py

@@ -13,11 +13,10 @@ from dkist_processing_common.tasks import WorkflowTaskBase
 from dkist_service_configuration.logging import logger
 
 from dkist_processing_dlnirsp.models.tags import DlnirspTag
-from dkist_processing_dlnirsp.tasks import AssembleDlnirspMovie
 from dkist_processing_dlnirsp.tasks import DlnirspAssembleQualityData
 from dkist_processing_dlnirsp.tasks import DlnirspL0QualityMetrics
 from dkist_processing_dlnirsp.tasks import DlnirspL1QualityMetrics
-from dkist_processing_dlnirsp.tasks import MakeDlnirspMovieFrames
+from dkist_processing_dlnirsp.tasks import MakeDlnirspMovie
 from dkist_processing_dlnirsp.tasks.bad_pixel_map import BadPixelCalibration
 from dkist_processing_dlnirsp.tasks.dark import DarkCalibration
 from dkist_processing_dlnirsp.tasks.geometric import GeometricCalibration
@@ -203,10 +202,11 @@ def main(
     load_bad_pixel: bool = False,
     load_geometric: bool = False,
     load_wavelength_calibration: bool = False,
-    load_solar: bool = False,
     load_inst_polcal: bool = False,
+    load_solar: bool = False,
     load_solar_gain_as_science: bool = False,
     load_calibrated_data: bool = False,
+    skip_movie: bool = False,
     force_intensity_only: bool = False,
     transfer_trial_data: str | None = None,
 ):
@@ -305,18 +305,18 @@ def main(
             manual_processing_run.run_task(task=WavelengthCalibration)
             manual_processing_run.run_task(task=SaveWavelengthCal)
 
-        if load_solar:
-            manual_processing_run.run_task(task=LoadSolarCal)
-        else:
-            manual_processing_run.run_task(task=SolarCalibration)
-            manual_processing_run.run_task(task=SaveSolarCal)
-
         if load_inst_polcal:
             manual_processing_run.run_task(task=LoadInstPolCal)
         else:
             manual_processing_run.run_task(task=InstrumentPolarizationCalibration)
             manual_processing_run.run_task(task=SaveInstPolCal)
 
+        if load_solar:
+            manual_processing_run.run_task(task=LoadSolarCal)
+        else:
+            manual_processing_run.run_task(task=SolarCalibration)
+            manual_processing_run.run_task(task=SaveSolarCal)
+
         if load_solar_gain_as_science:
             manual_processing_run.run_task(
                 task=load_solar_gain_as_science_task(force_intensity_only=force_intensity_only)
@@ -338,8 +338,6 @@ def main(
             task=permissive_write_l1_task(force_intensity_only=force_intensity_only)
         )
         manual_processing_run.run_task(task=ValidateL1Output)
-        manual_processing_run.run_task(task=MakeDlnirspMovieFrames)
-        manual_processing_run.run_task(task=AssembleDlnirspMovie)
 
         manual_processing_run.run_task(task=DlnirspL0QualityMetrics)
         manual_processing_run.run_task(task=DlnirspL1QualityMetrics)
@@ -348,6 +346,9 @@ def main(
 
         manual_processing_run.run_task(task=CreateTrialQualityReport)
 
+        if not skip_movie:
+            manual_processing_run.run_task(task=MakeDlnirspMovie)
+
         if transfer_trial_data:
             transfer_trial_data_locally(
                 trial_output_location=transfer_trial_data, processing_run=manual_processing_run
@@ -444,18 +445,18 @@ if __name__ == "__main__":
         help="Load wavelength calibration solution from previously saved run",
         action="store_true",
     )
-    parser.add_argument(
-        "-S",
-        "--load-solar",
-        help="Load solar calibration from previously saved run",
-        action="store_true",
-    )
     parser.add_argument(
         "-P",
         "--load-inst-polcal",
         help="Load instrument polarization calibration from previously saved run",
         action="store_true",
     )
+    parser.add_argument(
+        "-S",
+        "--load-solar",
+        help="Load solar calibration from previously saved run",
+        action="store_true",
+    )
     parser.add_argument(
         "-O",
         "--load-solar-gain-as-science",
@@ -465,6 +466,7 @@ if __name__ == "__main__":
     parser.add_argument(
         "-C", "--load-calibrated-data", help="Load CALIBRATED 'science' frames", action="store_true"
     )
+    parser.add_argument("-V", "--skip-movie", help="Don't make a browse movie", action="store_true")
     parser.add_argument(
         "-p",
         "--param-dir",
@@ -491,10 +493,11 @@ if __name__ == "__main__":
             load_bad_pixel=args.load_bad_pixel,
             load_geometric=args.load_geometric,
             load_wavelength_calibration=args.load_wavelength_calibration,
-            load_solar=args.load_solar,
             load_inst_polcal=args.load_inst_polcal,
+            load_solar=args.load_solar,
             load_solar_gain_as_science=args.load_solar_gain_as_science,
             load_calibrated_data=args.load_calibrated_data,
+            skip_movie=args.skip_movie,
             force_intensity_only=args.force_I_only,
             transfer_trial_data=args.transfer_trial_data,
         )

dkist_processing_dlnirsp/tests/local_trial_workflows/l0_to_l1.py

@@ -13,7 +13,6 @@ from dkist_processing_common.tasks import WorkflowTaskBase
 from dkist_service_configuration.logging import logger
 
 from dkist_processing_dlnirsp.models.tags import DlnirspTag
-from dkist_processing_dlnirsp.tasks.assemble_movie import AssembleDlnirspMovie
 from dkist_processing_dlnirsp.tasks.bad_pixel_map import BadPixelCalibration
 from dkist_processing_dlnirsp.tasks.dark import DarkCalibration
 from dkist_processing_dlnirsp.tasks.geometric import GeometricCalibration
@@ -22,7 +21,7 @@ from dkist_processing_dlnirsp.tasks.instrument_polarization import InstrumentPol
 from dkist_processing_dlnirsp.tasks.l1_output_data import DlnirspAssembleQualityData
 from dkist_processing_dlnirsp.tasks.lamp import LampCalibration
 from dkist_processing_dlnirsp.tasks.linearity_correction import LinearityCorrection
-from dkist_processing_dlnirsp.tasks.make_movie_frames import MakeDlnirspMovieFrames
+from dkist_processing_dlnirsp.tasks.movie import MakeDlnirspMovie
 from dkist_processing_dlnirsp.tasks.parse import ParseL0DlnirspLinearizedData
 from dkist_processing_dlnirsp.tasks.parse import ParseL0DlnirspRampData
 from dkist_processing_dlnirsp.tasks.quality_metrics import DlnirspL0QualityMetrics
@@ -171,9 +170,10 @@ def main(
     load_bad_pixel: bool = False,
     load_geometric: bool = False,
     load_wavelength_calibration: bool = False,
-    load_solar: bool = False,
     load_inst_polcal: bool = False,
+    load_solar: bool = False,
     load_calibrated_data: bool = False,
+    skip_movie: bool = False,
     transfer_trial_data: str | None = None,
 ):
     with ManualProcessing(
@@ -261,18 +261,18 @@ def main(
             manual_processing_run.run_task(task=WavelengthCalibration)
             manual_processing_run.run_task(task=SaveWavelengthCal)
 
-        if load_solar:
-            manual_processing_run.run_task(task=LoadSolarCal)
-        else:
-            manual_processing_run.run_task(task=SolarCalibration)
-            manual_processing_run.run_task(task=SaveSolarCal)
-
         if load_inst_polcal:
             manual_processing_run.run_task(task=LoadInstPolCal)
         else:
             manual_processing_run.run_task(task=InstrumentPolarizationCalibration)
             manual_processing_run.run_task(task=SaveInstPolCal)
 
+        if load_solar:
+            manual_processing_run.run_task(task=LoadSolarCal)
+        else:
+            manual_processing_run.run_task(task=SolarCalibration)
+            manual_processing_run.run_task(task=SaveSolarCal)
+
         if load_calibrated_data:
             manual_processing_run.run_task(task=LoadCalibratedData)
         else:
@@ -281,8 +281,6 @@ def main(
 
         manual_processing_run.run_task(task=DlnirspWriteL1Frame)
         manual_processing_run.run_task(task=ValidateL1Output)
-        manual_processing_run.run_task(task=MakeDlnirspMovieFrames)
-        manual_processing_run.run_task(task=AssembleDlnirspMovie)
 
         manual_processing_run.run_task(task=DlnirspL0QualityMetrics)
         manual_processing_run.run_task(task=DlnirspL1QualityMetrics)
@@ -293,6 +291,9 @@ def main(
 
         manual_processing_run.run_task(task=CreateTrialAsdf)
 
+        if not skip_movie:
+            manual_processing_run.run_task(task=MakeDlnirspMovie)
+
         if transfer_trial_data:
             transfer_trial_data_locally(
                 trial_output_location=transfer_trial_data, processing_run=manual_processing_run
@@ -391,21 +392,22 @@ if __name__ == "__main__":
         help="Load wavelength calibration solution from previously saved run",
         action="store_true",
     )
-    parser.add_argument(
-        "-S",
-        "--load-solar",
-        help="Load solar calibration from previously saved run",
-        action="store_true",
-    )
     parser.add_argument(
         "-P",
         "--load-inst-polcal",
         help="Load instrument polarization calibration from previously saved run",
         action="store_true",
     )
+    parser.add_argument(
+        "-S",
+        "--load-solar",
+        help="Load solar calibration from previously saved run",
+        action="store_true",
+    )
     parser.add_argument(
         "-C", "--load-calibrated-data", help="Load CALIBRATED science frames", action="store_true"
     )
+    parser.add_argument("-V", "--skip-movie", help="Don't make a browse movie", action="store_true")
     parser.add_argument(
         "-p",
         "--param-dir",
@@ -432,9 +434,10 @@ if __name__ == "__main__":
             load_bad_pixel=args.load_bad_pixel,
             load_geometric=args.load_geometric,
             load_wavelength_calibration=args.load_wavelength_calibration,
-            load_solar=args.load_solar,
             load_inst_polcal=args.load_inst_polcal,
+            load_solar=args.load_solar,
             load_calibrated_data=args.load_calibrated_data,
+            skip_movie=args.skip_movie,
             transfer_trial_data=args.transfer_trial_data,
         )
     )

dkist_processing_dlnirsp/tests/local_trial_workflows/local_trial_dev_mockers.py

@@ -89,7 +89,7 @@ def permissive_write_l1_task(force_intensity_only: bool):
             self, header: fits.Header, stokes: Literal["I", "Q", "U", "V"]
         ) -> fits.Header:
             if force_intensity_only:
-                header[MetadataKey.polarimeter_mode] = "Stokes I"
+                header[DlnirspMetadataKey.polarimeter_mode] = "Stokes I"
 
             return super().add_dataset_headers(header=header, stokes=stokes)
 

dkist_processing_dlnirsp/tests/test_dlnirsp_constants.py

@@ -28,6 +28,7 @@ class test_constants:
     arm_position_mm: float = 39.2
     grating_constant_inverse_mm: float = 19.0
     grating_position_deg: float = 87.4
+    obs_ip_end_time: str = "2000-01-01T00:01:00"
     # We don't need all the common ones, but let's put one just to check
     instrument: str = "CHECK_OUT_THIS_INSTRUMENT"
 
@@ -47,6 +48,7 @@ def simple_constant_names():
         "observe_exposure_times",
         "num_modstates",
         "solar_gain_ip_start_time",
+        "obs_ip_end_time",
         "instrument",
     ]
 

dkist_processing_dlnirsp/tests/test_movie.py

@@ -0,0 +1,141 @@
+import shutil
+
+import numpy as np
+import pytest
+from dkist_processing_common._util.scratch import WorkflowFileSystem
+from dkist_processing_common.codecs.json import json_encoder
+
+from dkist_processing_dlnirsp.models.tags import DlnirspTag
+from dkist_processing_dlnirsp.tasks.movie import MakeDlnirspMovie
+from dkist_processing_dlnirsp.tests.conftest import CalibratedHeaders
+from dkist_processing_dlnirsp.tests.conftest import DlnirspTestingConstants
+from dkist_processing_dlnirsp.tests.conftest import DlnirspTestingParameters
+from dkist_processing_dlnirsp.tests.conftest import write_calibrated_frames_to_task
+
+
+@pytest.fixture
+def movie_task(recipe_run_id, tmp_path, link_constants_db, assign_input_dataset_doc_to_task):
+    link_constants_db(
+        recipe_run_id=recipe_run_id,
+        constants_obj=DlnirspTestingConstants(),
+    )
+    with MakeDlnirspMovie(
+        recipe_run_id=recipe_run_id,
+        workflow_name="workflow_name",
+        workflow_version="workflow_version",
+    ) as task:
+        assign_input_dataset_doc_to_task(
+            task,
+            DlnirspTestingParameters(),
+            arm_id="VIS",
+        )
+        task.scratch = WorkflowFileSystem(scratch_base_path=tmp_path, recipe_run_id=recipe_run_id)
+
+        yield task
+        task._purge()
+
+
+@pytest.fixture(scope="session")
+def num_spectral_pix() -> int:
+    return 396
+
+
+@pytest.fixture(scope="session")
+def valid_wavelength_solution_header(num_spectral_pix) -> dict[str, str | int | float]:
+    """A valid wavelength solution header that depends on `arm_id='vis'` in `movie_task`."""
+    return {
+        "CTYPE3": "AWAV-GRA",
+        "CUNIT3": "nm",
+        "CRPIX3": num_spectral_pix // 2,
+        "CRVAL3": 854.17,
+        "CDELT3": 0.00229,
+        "PV3_0": 23000,
+        "PV3_1": 90,
+        "PV3_2": 65.69,
+    }
+
+
+def mindices_data_func(frame: CalibratedHeaders) -> np.ndarray:
+    shape = frame.array_shape
+
+    return np.ones(shape) * (1000 * frame.current_MINDEX1_value + 100 * frame.current_MINDEX2_value)
+
+
+@pytest.mark.parametrize(
+    "has_multiple_mosaics",
+    [pytest.param(True, id="multi_mosaic"), pytest.param(False, id="single_mosaic")],
+)
+@pytest.mark.parametrize(
+    "is_polarimetric",
+    [pytest.param(True, id="polarimetric"), pytest.param(False, id="spectrographic")],
+)
+@pytest.mark.parametrize(
+    "is_mosaiced", [pytest.param(True, id="mosaiced"), pytest.param(False, id="no_mosaic")]
+)
+def test_movie_task(
+    movie_task,
+    has_multiple_mosaics,
+    is_polarimetric,
+    is_mosaiced,
+    num_spectral_pix,
+    valid_wavelength_solution_header,
+    link_constants_db,
+    mocker,
+    fake_gql_client,
+):
+    """
+    Given: A `MakeDlnirspMovie` task with CALIBRATED frames
+    When: Running the task
+    Then: The dang thing runs and produces the expected movie file
+    """
+    mocker.patch(
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
+    )
+
+    num_dither = 1
+    num_mosaics = 2 if has_multiple_mosaics else 1
+    num_X_tiles = 2 if is_mosaiced else 1
+    num_Y_tiles = 2 if is_mosaiced else 1
+    stokes_params = ["I"]
+    if is_polarimetric:
+        stokes_params += ["Q", "U", "V"]
+        num_modstates = 8
+        pol_mode = "Full Stokes"
+    else:
+        num_modstates = 1
+        pol_mode = "Stokes I"
+
+    task = movie_task
+
+    constants_db = DlnirspTestingConstants(
+        POLARIMETER_MODE=pol_mode,
+        NUM_DITHER_STEPS=num_dither,
+        NUM_MOSAIC_REPEATS=num_mosaics,
+        NUM_MOSAIC_TILES_X=num_X_tiles,
+        NUM_MOSAIC_TILES_Y=num_Y_tiles,
+        NUM_MODSTATES=num_modstates,
+    )
+    link_constants_db(task.recipe_run_id, constants_db)
+
+    write_calibrated_frames_to_task(
+        task,
+        num_mosaics=num_mosaics,
+        num_X_tiles=num_X_tiles,
+        num_Y_tiles=num_Y_tiles,
+        dither_mode_on=False,
+        is_polarimetric=is_polarimetric,
+        array_shape=(num_spectral_pix, 20, 30),
+        data_func=mindices_data_func,
+    )
+
+    task.write(
+        data=valid_wavelength_solution_header,
+        tags=[DlnirspTag.intermediate(), DlnirspTag.task_wavelength_solution()],
+        encoder=json_encoder,
+    )
+
+    task()
+
+    movie_file_list = list(task.read(tags=[DlnirspTag.output(), DlnirspTag.movie()]))
+
+    assert len(movie_file_list) == 1

dkist_processing_dlnirsp/tests/test_parameters.py

@@ -53,6 +53,12 @@ def arm_parameter_names() -> list[str]:
         "ifu_y_pos_file_vis",
         "ifu_y_pos_file_jband",
         "ifu_y_pos_file_hband",
+        "movie_core_wave_value_nm_vis",
+        "movie_core_wave_value_nm_jband",
+        "movie_core_wave_value_nm_hband",
+        "movie_cont_wave_value_nm_vis",
+        "movie_cont_wave_value_nm_jband",
+        "movie_cont_wave_value_nm_hband",
     ]
 
 
@@ -170,6 +176,8 @@ def test_standard_parameters(
                 np.testing.assert_array_equal(param_obj_value, pv)
             elif isinstance(param_obj_value, BaseModel):
                 assert param_obj_value.model_dump() == pv
+            elif property_name == "movie_vertical_nan_slices":
+                assert [slice(*i) for i in pv] == param_obj_value
             else:
                 assert param_obj_value == pv
 

dkist_processing_dlnirsp/tests/test_parse.py

@@ -1,3 +1,5 @@
+import datetime
+
 import numpy as np
 import pytest
 from astropy.io import fits
@@ -255,6 +257,8 @@ def test_parse_linearized_data(
         grating_constant=grating_constant * 300,
         grating_angle=grating_angle * -6,
     )
+    obs_start_time = "2020-01-01T01:23:45"
+    modstate_length_sec = 0.5
     num_obs = write_observe_frames_to_task(
         task,
         num_modstates=num_mod,
@@ -267,8 +271,13 @@ def test_parse_linearized_data(
         grating_constant=grating_constant,
         grating_angle=grating_angle,
         dither_mode_on=dither_mode_on,
+        start_date=obs_start_time,
+        modstate_length_sec=modstate_length_sec,
         swap_crpix_values="swap" in task.parameters.wcs_crpix_correction_method,
     )
+    obs_end_time = datetime.datetime.fromisoformat(obs_start_time) + datetime.timedelta(
+        seconds=num_obs * modstate_length_sec
+    )
 
     task()
 
@@ -350,6 +359,7 @@ def test_parse_linearized_data(
     assert task.constants._db_dict[BudName.solar_gain_gos_level3_status] == "clear"
     assert task.constants._db_dict[BudName.solar_gain_num_raw_frames_per_fpa] == 30
     assert task.constants._db_dict[BudName.polcal_num_raw_frames_per_fpa] == 10
+    assert task.constants._db_dict[DlnirspBudName.obs_ip_end_time] == obs_end_time.isoformat("T")
 
 
 def test_crpix_and_spatial_step_association_swapped(

dkist_processing_dlnirsp/tests/test_science.py

@@ -70,15 +70,6 @@ def make_solar_data(solar_signal):
     return make_array
 
 
-@pytest.fixture
-def make_full_demodulation_matrix(demodulation_matrix):
-    def make_array(frame: Spec122Dataset):
-        array_shape = frame.array_shape[1:]
-        return np.ones(array_shape + demodulation_matrix.shape) * demodulation_matrix
-
-    return make_array
-
-
 @pytest.fixture
 def make_linearized_science_data(
     dark_signal,