dkist-processing-visp 2.20.14__py3-none-any.whl → 5.1.1__py3-none-any.whl

dkist_processing_visp/tests/test_quality.py

@@ -6,7 +6,6 @@ from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.fits import fits_array_encoder
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.models.task_name import TaskName
-from dkist_processing_common.tests.conftest import FakeGQLClient
 
 from dkist_processing_visp.models.tags import VispTag
 from dkist_processing_visp.tasks.quality_metrics import VispL0QualityMetrics
@@ -92,7 +91,7 @@ def visp_l1_quality_task(tmp_path, pol_mode, recipe_run_id, init_visp_constants_
     with VispL1QualityMetrics(
         recipe_run_id=recipe_run_id, workflow_name="science_calibration", workflow_version="VX.Y"
     ) as task:
-        task.scratch = WorkflowFileSystem(scratch_base_path=tmp_path)
+        task.scratch = WorkflowFileSystem(scratch_base_path=tmp_path, recipe_run_id=recipe_run_id)
 
         yield task, num_map_scans, num_raster_steps, num_stokes
         task._purge()
@@ -132,7 +131,7 @@ def test_l0_quality_task(
 
 
 @pytest.mark.parametrize("pol_mode", ["observe_polarimetric", "observe_intensity"])
-def test_l1_quality_task(visp_l1_quality_task, pol_mode, mocker):
+def test_l1_quality_task(visp_l1_quality_task, pol_mode, mocker, fake_gql_client):
     """
     Given: A VispL1QualityMetrics task
     When: Calling the task instance
@@ -140,7 +139,7 @@ def test_l1_quality_task(visp_l1_quality_task, pol_mode, mocker):
             and a single noise measurement and datetime is recorded for L1 file for each Stokes Q, U, and V
     """
     mocker.patch(
-        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
     )
     # When
     task, num_maps, num_steps, num_stokes = visp_l1_quality_task

dkist_processing_visp/tests/test_science.py

@@ -12,8 +12,8 @@ from dkist_header_validator import spec122_validator
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.fits import fits_array_encoder
 from dkist_processing_common.codecs.fits import fits_hdu_decoder
+from dkist_processing_common.models.fits_access import MetadataKey
 from dkist_processing_common.models.tags import Tag
-from dkist_processing_common.tests.conftest import FakeGQLClient
 
 from dkist_processing_visp.models.tags import VispStemName
 from dkist_processing_visp.models.tags import VispTag
@@ -144,7 +144,7 @@ def dummy_calibration_collection():
 
     dark_dict = {VispTag.beam(beam): {VispTag.readout_exp_time(0.04): np.zeros(intermediate_shape)}}
     background_dict = {VispTag.beam(beam): np.zeros(intermediate_shape)}
-    solar_dict = {VispTag.beam(beam): {VispTag.modstate(modstate): np.ones(intermediate_shape)}}
+    solar_dict = {VispTag.beam(beam): np.ones(intermediate_shape)}
     angle_dict = {VispTag.beam(beam): 0.0}
     spec_dict = {VispTag.beam(beam): np.zeros(intermediate_shape[1])}
     offset_dict = {VispTag.beam(beam): {VispTag.modstate(modstate): np.zeros(2)}}
@@ -184,7 +184,7 @@ def headers_with_dates() -> tuple[list[fits.Header], str, int, int]:
     ]
     random.shuffle(headers)  # Shuffle to make sure they're not already in time order
     for h in headers:
-        h["XPOSURE"] = exp_time  # Exposure time, in ms
+        h[MetadataKey.fpa_exposure_time_ms] = exp_time  # Exposure time, in ms
 
     return headers, start_time, exp_time, time_delta
 
@@ -252,10 +252,14 @@ def calibration_collection_with_full_overlap_slice() -> CalibrationCollection:
 
 @pytest.mark.parametrize(
     "background_on",
-    [pytest.param(True, id="Background on"), pytest.param(False, id="Background off")],
+    [pytest.param(True, id="background_on"), pytest.param(False, id="background_off")],
 )
 def test_science_calibration_task(
-    science_calibration_task, background_on, assign_input_dataset_doc_to_task, mocker
+    science_calibration_task,
+    background_on,
+    assign_input_dataset_doc_to_task,
+    mocker,
+    fake_gql_client,
 ):
     """
     Given: A ScienceCalibration task
@@ -264,7 +268,7 @@ def test_science_calibration_task(
     """
 
     mocker.patch(
-        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
     )
 
     # When
@@ -304,7 +308,8 @@ def test_science_calibration_task(
         task=task, num_modstates=num_modstates, data_shape=intermediate_shape, offsets=offsets
     )
     write_dummy_intermediate_solar_cals_to_task(
-        task=task, data_shape=intermediate_shape, num_modstates=num_modstates
+        task=task,
+        data_shape=intermediate_shape,
     )
     write_demod_matrices_to_task(task=task, num_modstates=num_modstates)
     write_input_observe_frames_to_task(
@@ -360,10 +365,10 @@ def test_science_calibration_task(
         assert header["VSPMAP"] == map_scan
 
         # Check that WCS keys were updated
-        if offsets[1, 0, 0] > 0:
-            assert header["CRPIX2"] == input_header["CRPIX2"] - np.ceil(offsets[1, 0, 0])
-        if offsets[1, 0, 1] > 0:
-            assert header["CRPIX1"] == input_header["CRPIX1"] - np.ceil(offsets[1, 0, 1])
+        if offsets[1, 0, 0] < 0:
+            assert header["CRPIX2"] == input_header["CRPIX2"] - np.ceil(-offsets[1, 0, 0])
+        if offsets[1, 0, 1] < 0:
+            assert header["CRPIX1"] == input_header["CRPIX1"] - np.ceil(-offsets[1, 0, 1])
 
     quality_files = task.read(tags=[Tag.quality("TASK_TYPES")])
     for file in quality_files:
@@ -416,7 +421,7 @@ def test_readout_normalization_correct(
     )
 
     # When:
-    corrected_array, _ = task.correct_single_frame(
+    corrected_array, _, _ = task.correct_single_frame(
         beam=1,
         modstate=1,
         raster_step=1,
@@ -509,7 +514,7 @@ def test_compute_date_keys_compressed_headers(
                     [[1.0, 2.0], [11.0, 10.0], [3.0, 2.0]],  # Beam 2
                 ]
             ),
-            [slice(11, None, None), slice(10, None, None)],
+            [slice(0, -11, None), slice(0, -10, None)],
         ),
         (
             np.array(
@@ -518,7 +523,7 @@ def test_compute_date_keys_compressed_headers(
                     [[-1.0, -2.0], [-11.0, -10.0], [-3.0, -2.0]],  # Beam 2
                 ]
             ),
-            [slice(0, -11, None), slice(0, -10, None)],
+            [slice(11, None, None), slice(10, None, None)],
         ),
         (
             np.array(
@@ -527,7 +532,7 @@ def test_compute_date_keys_compressed_headers(
                     [[1.0, 2.0], [-11.0, 10.0], [-3.0, -2.0]],  # Beam 2
                 ]
             ),
-            [slice(10, -11, None), slice(10, -2, None)],
+            [slice(11, -10, None), slice(2, -10, None)],
         ),
     ],
     ids=["All positive", "All negative", "Positive and negative"],
@@ -577,3 +582,96 @@ def test_combine_beams(
         expected = np.ones((10, 10, 4)) * 2.5
 
     np.testing.assert_array_equal(data, expected)
+
+
+@pytest.mark.parametrize(
+    "shifts",
+    # Shifts have shape (num_beams, num_modstates, 2)
+    # So the inner-most lists below (e.g., [5.0, 6.0]) correspond to [x_shift, y_shit]
+    [
+        np.array(
+            [
+                [[0.0, 0.0], [10.0, 2.0], [5.0, 6.0]],  # Beam 1
+                [[1.0, 2.0], [-11.0, 10.0], [-3.0, -2.0]],  # Beam 2
+            ]
+        ),
+    ],
+    ids=["Positive and negative"],
+)
+def test_combine_and_cut_nan_masks(
+    science_calibration_task, calibration_collection_with_geo_shifts, shifts
+):
+    """
+    Given: A ScienceCalibration task and NaN masks, along with geometric shifts
+    When: Combining the two NaN masks
+    Then: The final mask has NaN values in the correct place and is correctly cropped
+    """
+    nan_1_location = [0, 1]
+    nan_2_location = [50, 50]
+    nan_3_location = [4, 1]
+    nan_4_location = [55, 63]
+    nan_mask_shape = (100, 100)
+    nan_mask_1 = np.zeros(shape=nan_mask_shape)
+    nan_mask_1[nan_1_location[0], nan_1_location[1]] = np.nan
+    nan_mask_1[nan_2_location[0], nan_2_location[1]] = np.nan
+    nan_mask_2 = np.zeros(shape=nan_mask_shape)
+    nan_mask_2[nan_3_location[0], nan_3_location[1]] = np.nan
+    nan_mask_2[nan_4_location[0], nan_4_location[1]] = np.nan
+    task, _, _, _, _, _ = science_calibration_task
+    combined_nan_mask = task.combine_and_cut_nan_masks(
+        nan_masks=[nan_mask_1, nan_mask_2], calibrations=calibration_collection_with_geo_shifts
+    )
+    beam_1_shifts = shifts[0]
+    beam_2_shifts = shifts[1]
+    beam_1_x_shifts = [i[0] for i in beam_1_shifts]
+    beam_2_x_shifts = [i[0] for i in beam_2_shifts]
+    beam_1_y_shifts = [i[1] for i in beam_1_shifts]
+    beam_2_y_shifts = [i[1] for i in beam_2_shifts]
+    x_shifts = beam_1_x_shifts + beam_2_x_shifts
+    y_shifts = beam_1_y_shifts + beam_2_y_shifts
+    assert combined_nan_mask.shape == (
+        nan_mask_shape[0] - (max(x_shifts) - min(x_shifts)),
+        nan_mask_shape[1] - (max(y_shifts) - min(y_shifts)),
+    )
+    # Check that one NaN value from each original mask is present in the combined mask and in the correct place
+    assert (
+        combined_nan_mask[
+            nan_2_location[0] - int(abs(min(x_shifts))), nan_2_location[1] - int(abs(min(y_shifts)))
+        ]
+        == True
+    )
+    assert (
+        combined_nan_mask[
+            nan_4_location[0] - int(abs(min(x_shifts))), nan_4_location[1] - int(abs(min(y_shifts)))
+        ]
+        == True
+    )
+    assert np.sum(combined_nan_mask) == 2  # only two NaN values are in the final mask
+
+
+def test_generate_nan_mask(science_calibration_task, dummy_calibration_collection):
+    """
+    Given: a calibration collection
+    When: calculating the NaN mask to use
+    Then: the mask takes up some, but not all, of the frame size
+    """
+    task, _, _, _, _, _ = science_calibration_task
+    calibration_collection, _, _ = dummy_calibration_collection
+    beam = 1
+    modstate = 1
+    solar_gain_array = calibration_collection.solar_gain[VispTag.beam(beam)]
+    angle = calibration_collection.angle[VispTag.beam(beam)]
+    spec_shift = calibration_collection.spec_shift[VispTag.beam(beam)]
+    state_offset = calibration_collection.state_offset[VispTag.beam(beam)][
+        VispTag.modstate(modstate)
+    ]
+    nan_mask = task.generate_nan_mask(
+        solar_corrected_array=np.random.random(size=solar_gain_array.shape),
+        state_offset=state_offset,
+        angle=angle,
+        spec_shift=spec_shift,
+    )
+    # Some of the mask is marked as NaN but not all
+    assert np.sum(nan_mask) < np.size(nan_mask)
+    # Ensure that only zeroes and ones are in the mask
+    assert set(np.unique(nan_mask)) == {0, 1}