dkist-processing-common 10.8.2__tar.gz → 10.8.3__tar.gz

{dkist_processing_common-10.8.2 → dkist_processing_common-10.8.3}/CHANGELOG.rst

@@ -1,3 +1,18 @@
+v10.8.3 (2025-05-28)
+====================
+
+Features
+--------
+
+- Further constrain when the ATMOS_R0 header key is written in L1 data based on the following metrics:
+
+  * the ATMOS_R0 value in the L0 data must be less than or equal to 30cm
+  * the AO_LOCK value in the L0 data must be True
+  * the OOBSHIFT value in the L0 data must be less than 100
+
+  The wavefront sensor has about 1500 subapertures from which x and y shifts are measured. When the adaptive optics loop is locked, we count how many of these shifts exceed a threshold considered to be out of bounds. The r0 measurement gets less accurate as more of these shifts are out of bounds and so this is another way to detect outliers in the r0 value. (`#250 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/250>`__)
+
+
 v10.8.2 (2025-05-27)
 ====================
 

{dkist_processing_common-10.8.2 → dkist_processing_common-10.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 10.8.2
+Version: 10.8.3
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

dkist_processing_common-10.8.3/dkist_processing_common/models/fried_parameter.py

@@ -0,0 +1,41 @@
+"""Helper methods to handle fried parameter / r0 validity."""
+
+
+def r0_valid(
+    r0: float | None = None,
+    ao_lock: bool | None = None,
+    num_out_of_bounds_ao_values: int | None = None,
+) -> bool:
+    """
+    Determine if the r0 value should be considered valid based on the following conditions.
+
+        * ATMOS_R0 does not exist in the header.
+        * the value of ATMOS_R0 is greater than 0.3m
+        * the AO is not locked
+        * the value of OOBSHIFT is greater than 100
+
+    When the adaptive optics system is not locked, the ATMOS_R0 keyword is still filled with the output of the
+    Fried parameter calculation. The inputs are not valid in this instance and the value should be removed.
+
+    Sometimes, due to timing differences between the calculation of the Fried parameter and the AO lock status being
+    updated, non-physical values can be recorded for ATMOS_R0 right on the edge of an AO_LOCK state change. To
+    combat this, any remaining R0 values greater than 30cm (which is beyond the realm of physical possibility for
+    solar observations) are also removed.
+
+    In addition, the number of AO out-of-bound values is given in the keyword OOBSHIFT and the AO team advises
+    that values under 100 are when the r0 value is considered reliable. If the OOBSHIFT key doesn't exist, this check
+    should be ignored.
+    """
+    if r0 is None:
+        return False
+
+    if r0 > 0.3:
+        return False
+
+    if ao_lock is not True:
+        return False
+
+    if num_out_of_bounds_ao_values is not None and num_out_of_bounds_ao_values > 100:
+        return False
+
+    return True

{dkist_processing_common-10.8.2 → dkist_processing_common-10.8.3}/dkist_processing_common/parsers/quality.py

@@ -31,3 +31,4 @@ class L1QualityFitsAccess(L1FitsAccess):
         self.light_level: float = self.header["LIGHTLVL"]
         self.health_status: str = self.header["DSHEALTH"]
         self.ao_status: int = self.header.get("AO_LOCK", None)
+        self.num_out_of_bounds_ao_values: int = self.header.get("OOBSHIFT", None)

{dkist_processing_common-10.8.2 → dkist_processing_common-10.8.3}/dkist_processing_common/tasks/mixin/quality/_metrics.py

@@ -20,6 +20,7 @@ from dkist_processing_pac.fitter.fitting_core import compare_I
 from dkist_processing_pac.fitter.polcal_fitter import PolcalFitter
 from pandas import DataFrame
 
+from dkist_processing_common.models.fried_parameter import r0_valid
 from dkist_processing_common.models.metric_code import MetricCode
 from dkist_processing_common.models.quality import EfficiencyHistograms
 from dkist_processing_common.models.quality import ModulationMatrixHistograms
@@ -203,7 +204,7 @@ class _SimplePlotQualityMixin:
         return warnings
 
     def quality_store_ao_status_and_fried_parameter(
-        self, datetimes: list[str], values: list[list[bool, float]]
+        self, datetimes: list[str], values: list[list[bool | float]]
     ):
         """
         Collect and store datetime / value pairs for the boolean AO status and Fried parameter.
@@ -213,23 +214,27 @@ class _SimplePlotQualityMixin:
         Because of how L1Metric.has_metric works, empty lists will not be passed to this method.
         However, because of how L1Metric.store_metric works, one or both values can be None.
         """
-        ao_values = [value[0] for value in values]
-        ao_not_none = [ao for ao in ao_values if ao is not None]
+        ao_lock_values = [value[0] for value in values]
+        ao_not_none = [ao for ao in ao_lock_values if ao is not None]
         if len(ao_not_none) != 0:
             self._record_values(values=ao_not_none, tags=Tag.quality(MetricCode.ao_status))
         fried_values = [value[1] for value in values]
-        fried_values_not_none_ao_locked = [
-            r0 for ao, r0 in zip(ao_values, fried_values) if (ao is True) and (r0 is not None)
-        ]
-        datetimes_fried_not_none_ao_locked = [
-            dt
-            for dt, ao, r0 in zip(datetimes, ao_values, fried_values)
-            if (ao is True) and (r0 is not None)
-        ]
-        if len(fried_values_not_none_ao_locked) != 0:
+        ao_oob_values = [value[2] for value in values]
+        fried_values_to_plot = []
+        datetimes_to_plot = []
+        # For each set of input data, check if the r0 is considered valid based on all data
+        for i in range(len(fried_values)):
+            if r0_valid(
+                r0=fried_values[i],
+                ao_lock=ao_lock_values[i],
+                num_out_of_bounds_ao_values=ao_oob_values[i],
+            ):
+                fried_values_to_plot.append(fried_values[i])
+                datetimes_to_plot.append(datetimes[i])
+        if len(fried_values_to_plot) != 0:
             self._record_2d_plot_values(
-                x_values=datetimes_fried_not_none_ao_locked,
-                y_values=fried_values_not_none_ao_locked,
+                x_values=datetimes_to_plot,
+                y_values=fried_values_to_plot,
                 tags=Tag.quality(MetricCode.fried_parameter),
             )
 

{dkist_processing_common-10.8.2 → dkist_processing_common-10.8.3}/dkist_processing_common/tasks/quality_metrics.py

@@ -296,7 +296,7 @@ class QualityL1Metrics(WorkflowTaskBase, QualityMixin):
             L1Metric(storage_method=self.quality_store_health_status, value_source="health_status"),
             L1Metric(
                 storage_method=self.quality_store_ao_status_and_fried_parameter,
-                value_source=["ao_status", "fried_parameter"],
+                value_source=["ao_status", "fried_parameter", "num_out_of_bounds_ao_values"],
             ),
         ]
 

{dkist_processing_common-10.8.2 → dkist_processing_common-10.8.3}/dkist_processing_common/tasks/write_l1.py

@@ -29,6 +29,7 @@ from sunpy.coordinates import Helioprojective
 
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
+from dkist_processing_common.models.fried_parameter import r0_valid
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.models.wavelength import WavelengthRange
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
@@ -373,8 +374,14 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         """
         # Replace header values in place
         header = self.replace_header_values(header=header, data=data)
-        # Remove r0 value if AO not locked
-        header = self.remove_invalid_r0_values(header=header)
+        # Remove r0 value if r0 conditions are not met
+        r0_is_valid = r0_valid(
+            r0=header["ATMOS_R0"],
+            ao_lock=header.get("AO_LOCK", None),
+            num_out_of_bounds_ao_values=header.get("OOBSHIFT", None),
+        )
+        if not r0_is_valid:
+            header.pop("ATMOS_R0", None)
         # Add the stats table
         header = self.add_stats_headers(header=header, data=data)
         # Add the datacenter table

dkist_processing_common-10.8.3/dkist_processing_common/tests/test_fried_parameter.py

@@ -0,0 +1,27 @@
+import pytest
+
+from dkist_processing_common.models.fried_parameter import r0_valid
+
+
+@pytest.mark.parametrize(
+    "r0, ao_lock, oob_shift, should_r0_exist",
+    [
+        pytest.param(0.2, True, 17, True, id="AO_LOCK_True_good_R0_good_oob"),
+        pytest.param(1, True, 17, False, id="AO_LOCK_True_bad_R0_good_oob"),
+        pytest.param(0.2, False, 17, False, id="AO_LOCK_False_good_R0_good_oob"),
+        pytest.param(1, False, 17, False, id="AO_LOCK_False_bad_R0_good_oob"),
+        pytest.param(0.2, True, 150, False, id="AO_LOCK_True_good_R0_bad_oob"),
+        pytest.param(1, True, 150, False, id="AO_LOCK_True_bad_R0_bad_oob"),
+        pytest.param(0.2, False, 150, False, id="AO_LOCK_False_good_R0_bad_oob"),
+        pytest.param(1, False, 150, False, id="AO_LOCK_False_bad_R0_bad_oob"),
+        pytest.param(0.2, None, 17, False, id="AO_LOCK_missing"),
+        pytest.param(0.2, True, None, True, id="OOBSHIFT_missing"),
+    ],
+)
+def test_check_r0_valid(r0, ao_lock, oob_shift, should_r0_exist):
+    """
+    :Given: values for r0, the ao_lock status, and the ao out of bound shift value
+    :When: checking for a valid state to use r0
+    :Then: valid conditions are marked True, invalid conditions marked False
+    """
+    assert r0_valid(r0, ao_lock, oob_shift) == should_r0_exist

{dkist_processing_common-10.8.2 → dkist_processing_common-10.8.3}/dkist_processing_common/tests/test_quality_mixin.py

@@ -39,9 +39,9 @@ def quality_task(tmp_path, recipe_run_id):
 @pytest.fixture
 def plot_data():
     datetimes_a = ["2021-01-01T01:01:01", "2021-01-01T02:01:01"]
-    values_a = [3, 4]
+    values_a = [0.1, 0.2]
     datetimes_b = ["2020-01-01T01:01:01", "2020-01-01T02:01:01"]
-    values_b = [1, 2]
+    values_b = [0.15, 0.25]
     return datetimes_a, values_a, datetimes_b, values_b
 
 
@@ -198,8 +198,9 @@ def test_store_ao_status_and_fried_parameter(quality_task, ao_values):
     """
     task = quality_task
     datetimes = ["2020-01-01T01:01:01", "2020-01-01T02:01:01"]
-    fried_values = [3.0, 4.0]
-    combined_values = [[ao, r0] for ao, r0 in zip(ao_values, fried_values)]
+    fried_values = [0.1, 0.2]
+    oob_values = [25, 50]
+    combined_values = [[ao, r0, oob] for ao, r0, oob in zip(ao_values, fried_values, oob_values)]
     task.quality_store_ao_status_and_fried_parameter(datetimes=datetimes, values=combined_values)
     path = list(task.read(tags=Tag.quality("AO_STATUS")))
     assert len(path) == 1
@@ -220,10 +221,11 @@ def test_store_ao_status_and_fried_parameter(quality_task, ao_values):
 @pytest.mark.parametrize(
     "combined_values",
     [
-        pytest.param([[True, 1], [None, 2]], id="AO_some_none"),
-        pytest.param([[True, 1], [True, None]], id="Fried_some_none"),
-        pytest.param([[None, 1], [None, 2]], id="AO_all_none"),
-        pytest.param([[True, None], [True, None]], id="Fried_all_none"),
+        pytest.param([[True, 0.1, 25], [None, 0.2, 25]], id="AO_some_none"),
+        pytest.param([[True, 0.1, 25], [True, None, 25]], id="Fried_some_none"),
+        pytest.param([[None, 0.1, 25], [None, 0.2, 25]], id="AO_all_none"),
+        pytest.param([[True, None, 25], [True, None, 25]], id="Fried_all_none"),
+        pytest.param([[True, 0.1, None], [True, 0.2, None]], id="Out_of_bounds_all_none"),
     ],
 )
 def test_store_ao_status_and_fried_parameter_with_nones(quality_task, combined_values):
@@ -231,13 +233,14 @@ def test_store_ao_status_and_fried_parameter_with_nones(quality_task, combined_v
     datetimes = ["2020-01-01T01:01:01", "2020-01-01T02:01:01"]
     task.quality_store_ao_status_and_fried_parameter(datetimes=datetimes, values=combined_values)
     path = list(task.read(tags=Tag.quality("AO_STATUS")))
-    ao_values = [ao for ao, r0 in combined_values]
-    fried_values = [r0 for ao, r0 in combined_values]
+    ao_values = [ao for ao, r0, oob in combined_values]
+    fried_values = [r0 for ao, r0, oob in combined_values]
+    ao_out_of_bounds = [oob for ao, r0, oob in combined_values]
     if not all(ao is None for ao in ao_values):
         assert len(path) == 1
         with path[0].open() as f:
             data = json.load(f)
-            assert len(data) == sum(1 for ao, r0 in combined_values if ao is not None)
+            assert len(data) == sum(1 for ao, r0, oob in combined_values if ao is not None)
     else:
         assert len(path) == 0
     path = list(task.read(tags=Tag.quality("FRIED_PARAMETER")))
@@ -246,7 +249,7 @@ def test_store_ao_status_and_fried_parameter_with_nones(quality_task, combined_v
         with path[0].open() as f:
             data = json.load(f)
             assert len(data["y_values"]) == sum(
-                1 for ao, r0 in combined_values if ao is True and r0 is not None
+                1 for ao, r0, oob in combined_values if ao is True and r0 is not None
             )
     else:
         assert len(path) == 0
@@ -263,7 +266,8 @@ def test_build_ao_status(quality_task, plot_data):
     datetimes = datetimes_a + datetimes_b
     fried_values = values_a + values_b
     ao_values = [False, True, True, True]
-    combined_values = [[ao, r0] for ao, r0 in zip(ao_values, fried_values)]
+    oob_values = [25, 50, None, 50]
+    combined_values = [[ao, r0, oob] for ao, r0, oob in zip(ao_values, fried_values, oob_values)]
     task.quality_store_ao_status_and_fried_parameter(datetimes=datetimes, values=combined_values)
     metric = task.quality_build_ao_status()
     assert metric["name"] == "Adaptive Optics Status"
@@ -284,9 +288,15 @@ def test_build_fried_parameter(quality_task, plot_data):
     task = quality_task
     datetimes_a, fried_values_a, datetimes_b, fried_values_b = plot_data
     ao_values_a = [True, True]
-    combined_values_a = [[ao, r0] for ao, r0 in zip(ao_values_a, fried_values_a)]
+    oob_values_a = [25, 50]
+    combined_values_a = [
+        [ao, r0, oob] for ao, r0, oob in zip(ao_values_a, fried_values_a, oob_values_a)
+    ]
     ao_values_b = [True, True]
-    combined_values_b = [[ao, r0] for ao, r0 in zip(ao_values_b, fried_values_b)]
+    oob_values_b = [25, 50]
+    combined_values_b = [
+        [ao, r0, oob] for ao, r0, oob in zip(ao_values_b, fried_values_b, oob_values_b)
+    ]
     task.quality_store_ao_status_and_fried_parameter(
         datetimes=datetimes_a, values=combined_values_a
     )
@@ -307,14 +317,14 @@ def test_build_fried_parameter(quality_task, plot_data):
             "2021-01-01T02:01:01",
         ]
     ]
-    assert metric["plot_data"]["series_data"][""][1] == [1, 2, 3, 4]
+    assert metric["plot_data"]["series_data"][""][1] == [0.15, 0.25, 0.1, 0.2]
     assert metric["name"] == "Fried Parameter"
     assert metric["metric_code"] == "FRIED_PARAMETER"
     assert metric["facet"] is None
     assert metric["warnings"] is None
     assert (
         metric["statement"]
-        == "Average valid Fried Parameter measurements for L1 dataset: 2.5 ± 1.12 m"
+        == "Average valid Fried Parameter measurements for L1 dataset: 0.18 ± 0.06 m"
     )
 
 
@@ -340,12 +350,12 @@ def test_build_light_level(quality_task, plot_data):
             "2021-01-01T02:01:01",
         ]
     ]
-    assert metric["plot_data"]["series_data"][""][1] == [1, 2, 3, 4]
+    assert metric["plot_data"]["series_data"][""][1] == [0.15, 0.25, 0.1, 0.2]
     assert metric["name"] == "Light Level"
     assert metric["metric_code"] == "LIGHT_LEVEL"
     assert metric["facet"] is None
     assert metric["warnings"] is None
-    assert metric["statement"] == f"Average Light Level for L1 dataset: 2.5 ± 1.12 adu"
+    assert metric["statement"] == f"Average Light Level for L1 dataset: 0.18 ± 0.06 adu"
 
 
 def test_build_frame_average(quality_task, plot_data):
@@ -461,7 +471,7 @@ def test_build_noise(quality_task, plot_data):
             "2021-01-01T02:01:01",
         ]
     ]
-    assert metric["plot_data"]["series_data"]["I"][1] == [1, 2, 3, 4]
+    assert metric["plot_data"]["series_data"]["I"][1] == [0.15, 0.25, 0.1, 0.2]
     assert metric["name"] == "Noise Estimation"
     assert metric["metric_code"] == "NOISE"
     assert metric["facet"] is None
@@ -493,7 +503,7 @@ def test_build_sensitivity(quality_task, plot_data):
             "2021-01-01T02:01:01",
         ]
     ]
-    assert metric["plot_data"]["series_data"]["I"][1] == [1, 2, 3, 4]
+    assert metric["plot_data"]["series_data"]["I"][1] == [0.15, 0.25, 0.1, 0.2]
     assert metric["name"] == f"Sensitivity"
     assert metric["metric_code"] == "SENSITIVITY"
     assert metric["facet"] is None
@@ -692,7 +702,7 @@ def test_build_report(quality_task, plot_data):
     task.quality_store_task_type_counts(task_type="dark", total_frames=100, frames_not_used=7)
     task.quality_store_task_type_counts(task_type="gain", total_frames=100, frames_not_used=0)
     task.quality_store_ao_status_and_fried_parameter(
-        datetimes=datetimes, values=[[True, values[0]], [True, values[1]]]
+        datetimes=datetimes, values=[[True, values[0], values[0]], [True, values[1], values[1]]]
     )
     task.quality_store_light_level(datetimes=datetimes, values=values)
     task.quality_store_frame_average(

{dkist_processing_common-10.8.2 → dkist_processing_common-10.8.3}/dkist_processing_common/tests/test_write_l1.py

@@ -201,9 +201,32 @@ def write_l1_task_with_empty_waveband(recipe_run_id, tmp_path, request):
 @pytest.fixture(
     scope="function",
     params=[
-        pytest.param({"AO_LOCK": True}, id="AO_LOCK_True"),
-        pytest.param({"AO_LOCK": False}, id="AO_LOCK_False"),
-        pytest.param({}, id="AO_LOCK_missing"),
+        pytest.param(
+            {"AO_LOCK": True, "ATMOS_R0": 0.2, "OOBSHIFT": 17}, id="AO_LOCK_True_good_R0_good_oob"
+        ),
+        pytest.param(
+            {"AO_LOCK": True, "ATMOS_R0": 1, "OOBSHIFT": 17}, id="AO_LOCK_True_bad_R0_good_oob"
+        ),
+        pytest.param(
+            {"AO_LOCK": False, "ATMOS_R0": 0.2, "OOBSHIFT": 17}, id="AO_LOCK_False_good_R0_good_oob"
+        ),
+        pytest.param(
+            {"AO_LOCK": False, "ATMOS_R0": 1, "OOBSHIFT": 17}, id="AO_LOCK_False_bad_R0_good_oob"
+        ),
+        pytest.param(
+            {"AO_LOCK": True, "ATMOS_R0": 0.2, "OOBSHIFT": 150}, id="AO_LOCK_True_good_R0_bad_oob"
+        ),
+        pytest.param(
+            {"AO_LOCK": True, "ATMOS_R0": 1, "OOBSHIFT": 150}, id="AO_LOCK_True_bad_R0_bad_oob"
+        ),
+        pytest.param(
+            {"AO_LOCK": False, "ATMOS_R0": 0.2, "OOBSHIFT": 150}, id="AO_LOCK_False_good_R0_bad_oob"
+        ),
+        pytest.param(
+            {"AO_LOCK": False, "ATMOS_R0": 1, "OOBSHIFT": 150}, id="AO_LOCK_False_bad_R0_bad_oob"
+        ),
+        pytest.param({"ATMOS_R0": 0.2, "OOBSHIFT": 17}, id="AO_LOCK_missing"),
+        pytest.param({"ATMOS_R0": 0.2, "AO_LOCK": True}, id="OOBSHIFT_missing"),
     ],
 )
 def write_l1_task_no_data(request, recipe_run_id, tmp_path, complete_common_header):
@@ -216,10 +239,10 @@ def write_l1_task_no_data(request, recipe_run_id, tmp_path, complete_common_head
     ):
         task.scratch = WorkflowFileSystem(recipe_run_id=recipe_run_id, scratch_base_path=tmp_path)
         header = complete_common_header
-        header.pop("AO_LOCK", None)  # If it's not required, shouldn't be here in the first place??
+        header.pop("AO_LOCK", None)
+        header.pop("ATMOS_R0", None)
+        header.pop("OOBSHIFT", None)
         header.update(request.param)
-        fried_parameter = 0.2
-        header["ATMOS_R0"] = fried_parameter
         hdu = fits.PrimaryHDU(data=np.random.random(size=(1, 1, 1)) * 1, header=header)
         hdul = fits.HDUList([hdu])
         task.write(
@@ -231,7 +254,9 @@ def write_l1_task_no_data(request, recipe_run_id, tmp_path, complete_common_head
             encoder=fits_hdulist_encoder,
         )
         task.constants._update(asdict(FakeConstantDb()))
-        yield task, header, fried_parameter
+        fried_parameter = request.param["ATMOS_R0"]
+        oob_shift = request.param.get("OOBSHIFT")
+        yield task, header, fried_parameter, oob_shift
         task._purge()
 
 
@@ -650,7 +675,7 @@ def test_check_r0_ao_lock(write_l1_task_no_data):
     :When: writing, check if the AO lock is on
     :Then: write the r0 value if AO lock on, don't write if AO lock off
     """
-    task, header, r0 = write_l1_task_no_data
+    task, header, r0, _ = write_l1_task_no_data
     header_after_check = task.remove_invalid_r0_values(header=header)
     if header.get("AO_LOCK"):
         assert header_after_check["ATMOS_R0"] == header["ATMOS_R0"]

{dkist_processing_common-10.8.2 → dkist_processing_common-10.8.3}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 10.8.2
+Version: 10.8.3
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-10.8.2 → dkist_processing_common-10.8.3}/dkist_processing_common.egg-info/SOURCES.txt

@@ -35,6 +35,7 @@ dkist_processing_common/models/__init__.py
 dkist_processing_common/models/constants.py
 dkist_processing_common/models/fits_access.py
 dkist_processing_common/models/flower_pot.py
+dkist_processing_common/models/fried_parameter.py
 dkist_processing_common/models/graphql.py
 dkist_processing_common/models/message.py
 dkist_processing_common/models/message_queue_binding.py
@@ -91,6 +92,7 @@ dkist_processing_common/tests/test_constants.py
 dkist_processing_common/tests/test_cs_step.py
 dkist_processing_common/tests/test_fits_access.py
 dkist_processing_common/tests/test_flower_pot.py
+dkist_processing_common/tests/test_fried_parameter.py
 dkist_processing_common/tests/test_input_dataset.py
 dkist_processing_common/tests/test_interservice_bus.py
 dkist_processing_common/tests/test_interservice_bus_mixin.py