dkist-processing-common 10.8.1rc1__tar.gz → 10.8.3__tar.gz

{dkist_processing_common-10.8.1rc1 → dkist_processing_common-10.8.3}/CHANGELOG.rst

@@ -1,3 +1,36 @@
+v10.8.3 (2025-05-28)
+====================
+
+Features
+--------
+
+- Further constrain when the ATMOS_R0 header key is written in L1 data based on the following metrics:
+
+  * the ATMOS_R0 value in the L0 data must be less than or equal to 30cm
+  * the AO_LOCK value in the L0 data must be True
+  * the OOBSHIFT value in the L0 data must be less than 100
+
+  The wavefront sensor has about 1500 subapertures from which x and y shifts are measured. When the adaptive optics loop is locked, we count how many of these shifts exceed a threshold considered to be out of bounds. The r0 measurement gets less accurate as more of these shifts are out of bounds and so this is another way to detect outliers in the r0 value. (`#250 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/250>`__)
+
+
+v10.8.2 (2025-05-27)
+====================
+
+Bugfixes
+--------
+
+- Prevent the `WAVEBAND` key from being populated with a spectral line that falls outside the wavelength bounds of the data. (`#251 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/251>`__)
+
+
+v10.8.1 (2025-05-22)
+====================
+
+Misc
+----
+
+- Update `dkist-processing-core` to v5.1.1. (`#253 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/253>`__)
+
+
 v10.8.0 (2025-05-15)
 ====================
 

{dkist_processing_common-10.8.1rc1 → dkist_processing_common-10.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 10.8.1rc1
+Version: 10.8.3
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -17,7 +17,7 @@ Requires-Dist: asdf<4.0.0,>=3.5.0
 Requires-Dist: astropy>=7.0.0
 Requires-Dist: dkist-fits-specifications<5.0,>=4.0.0
 Requires-Dist: dkist-header-validator<6.0,>=5.0.0
-Requires-Dist: dkist-processing-core==5.1.0
+Requires-Dist: dkist-processing-core==5.1.1
 Requires-Dist: dkist-processing-pac<4.0,>=3.1
 Requires-Dist: dkist-service-configuration<3.0,>=2.0.2
 Requires-Dist: dkist-spectral-lines<4.0,>=3.0.0

{dkist_processing_common-10.8.1rc1 → dkist_processing_common-10.8.3}/dkist_processing_common/codecs/fits.py

@@ -30,15 +30,15 @@ def fits_hdulist_encoder(hdu_list: fits.HDUList) -> bytes:
     return iobase_encoder(file_obj)
 
 
-def fits_hdu_decoder(path: Path, hdu: int | None = None) -> fits.PrimaryHDU | fits.CompImageHDU:
+def fits_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
     """Read a Path with `fits` to produce an `HDUList`."""
     hdu_list = fits.open(path, checksum=True)
-    return _extract_hdu(hdu_list, hdu)
+    return _extract_hdu(hdu_list)
 
 
-def fits_array_decoder(path: Path, hdu: int | None = None, auto_squeeze: bool = True) -> np.ndarray:
+def fits_array_decoder(path: Path, auto_squeeze: bool = True) -> np.ndarray:
     """Read a Path with `fits` and return the `.data` property."""
-    hdu = fits_hdu_decoder(path, hdu=hdu)
+    hdu = fits_hdu_decoder(path)
     data = hdu.data
 
     # This conditional is explicitly to catch summit data with a dummy first axis for WCS
@@ -56,14 +56,8 @@ def fits_access_decoder(
     return fits_access_class(hdu=hdu, name=str(path), **fits_access_kwargs)
 
 
-def _extract_hdu(hdul: fits.HDUList, hdu: int | None = None) -> fits.PrimaryHDU | fits.CompImageHDU:
-    """
-    Return the fits hdu associated with the data in the hdu list.
-
-    Only search down the hdu index for the data if the hdu index is not explicitly provided.
-    """
-    if hdu is not None:
-        return hdul[hdu]
+def _extract_hdu(hdul: fits.HDUList) -> fits.PrimaryHDU | fits.CompImageHDU:
+    """Return the fits hdu associated with the data in the hdu list."""
     if hdul[0].data is not None:
         return hdul[0]
     return hdul[1]

{dkist_processing_common-10.8.1rc1 → dkist_processing_common-10.8.3}/dkist_processing_common/manual.py

@@ -2,13 +2,15 @@
 import json
 import logging
 import shutil
+from dataclasses import asdict
+from io import BytesIO
 from pathlib import Path
 from typing import Callable
 from unittest.mock import patch
 
 from dkist_processing_core.task import TaskBase
 
-from dkist_processing_common.codecs.basemodel import basemodel_encoder
+from dkist_processing_common.codecs.json import json_encoder
 from dkist_processing_common.models.graphql import RecipeRunProvenanceMutation
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.base import WorkflowTaskBase
@@ -180,8 +182,8 @@ def writing_metadata_store_record_provenance(self, is_task_manual: bool, library
         workflowVersion=self.workflow_version,
     )
     self.write(
-        data=params,
-        encoder=basemodel_encoder,
+        data=params.model_dump(),
+        encoder=json_encoder,
         tags=["PROVENANCE_RECORD"],
         relative_path=f"{self.task_name}_provenance.json",
         overwrite=True,

dkist_processing_common-10.8.3/dkist_processing_common/models/fried_parameter.py

@@ -0,0 +1,41 @@
+"""Helper methods to handle fried parameter / r0 validity."""
+
+
+def r0_valid(
+    r0: float | None = None,
+    ao_lock: bool | None = None,
+    num_out_of_bounds_ao_values: int | None = None,
+) -> bool:
+    """
+    Determine if the r0 value should be considered valid based on the following conditions.
+
+        * ATMOS_R0 does not exist in the header.
+        * the value of ATMOS_R0 is greater than 0.3m
+        * the AO is not locked
+        * the value of OOBSHIFT is greater than 100
+
+    When the adaptive optics system is not locked, the ATMOS_R0 keyword is still filled with the output of the
+    Fried parameter calculation. The inputs are not valid in this instance and the value should be removed.
+
+    Sometimes, due to timing differences between the calculation of the Fried parameter and the AO lock status being
+    updated, non-physical values can be recorded for ATMOS_R0 right on the edge of an AO_LOCK state change. To
+    combat this, any remaining R0 values greater than 30cm (which is beyond the realm of physical possibility for
+    solar observations) are also removed.
+
+    In addition, the number of AO out-of-bound values is given in the keyword OOBSHIFT and the AO team advises
+    that values under 100 are when the r0 value is considered reliable. If the OOBSHIFT key doesn't exist, this check
+    should be ignored.
+    """
+    if r0 is None:
+        return False
+
+    if r0 > 0.3:
+        return False
+
+    if ao_lock is not True:
+        return False
+
+    if num_out_of_bounds_ao_values is not None and num_out_of_bounds_ao_values > 100:
+        return False
+
+    return True

{dkist_processing_common-10.8.1rc1 → dkist_processing_common-10.8.3}/dkist_processing_common/models/graphql.py

@@ -3,9 +3,6 @@ from pydantic import BaseModel
 from pydantic import field_validator
 from pydantic import Json
 
-from dkist_processing_common.models.input_dataset import InputDatasetBaseModel
-from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
-
 
 class RecipeRunMutation(BaseModel):
     """Recipe run mutation record."""
@@ -40,19 +37,13 @@ class InputDatasetPartTypeResponse(BaseModel):
     inputDatasetPartTypeName: str
 
 
-class InputDatasetPartResponse(InputDatasetBaseModel):
+class InputDatasetPartResponse(BaseModel):
     """Response class for the input dataset part entity."""
 
     inputDatasetPartId: int
-    # inputDatasetPartDocument : Json[InputDatasetPartDocumentList] # will work in gqlclient v2
-    inputDatasetPartDocument: Json[list]
+    inputDatasetPartDocument: Json[list[dict]]
     inputDatasetPartType: InputDatasetPartTypeResponse
 
-    @field_validator("inputDatasetPartDocument", mode="after")
-    @classmethod
-    def _use_frame_or_parameter_model(cls, value_list):  # not needed for gqlclient v2
-        return InputDatasetPartDocumentList(doc_list=value_list)
-
 
 class InputDatasetInputDatasetPartResponse(BaseModel):
     """Response class for the join entity between input datasets and input dataset parts."""
@@ -112,12 +103,11 @@ class RecipeRunResponse(BaseModel):
     recipeInstance: RecipeInstanceResponse
     recipeInstanceId: int
     recipeRunProvenances: list[RecipeRunProvenanceResponse]
-    # configuration: Json[RecipeRunConfiguration] | None # will work in gqlclient v2
     configuration: Json[dict] | None
 
     @field_validator("configuration", mode="after")
     @classmethod
-    def _use_recipe_run_configuration_model(cls, value):  # not needed for gqlclient v2
+    def _use_recipe_run_configuration_model(cls, value):
         if value is None:
             return RecipeRunConfiguration()
         return RecipeRunConfiguration.model_validate(value)

{dkist_processing_common-10.8.1rc1 → dkist_processing_common-10.8.3}/dkist_processing_common/models/parameters.py

@@ -1,23 +1,14 @@
 """Base class for parameter-parsing object."""
 import logging
-from contextlib import contextmanager
 from datetime import datetime
-from pathlib import Path
 from typing import Any
-from typing import Callable
 from typing import Literal
 
 import numpy as np
 import scipy.interpolate as spi
+from astropy.io import fits
 
-from dkist_processing_common._util.scratch import WorkflowFileSystem
-from dkist_processing_common.codecs.array import array_decoder
-from dkist_processing_common.codecs.basemodel import basemodel_decoder
-from dkist_processing_common.codecs.fits import fits_array_decoder
-from dkist_processing_common.models.input_dataset import InputDatasetFilePointer
-from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
-from dkist_processing_common.models.tags import Tag
-
+from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetParameterValue
 
 logger = logging.getLogger(__name__)
 
@@ -33,9 +24,9 @@ class ParameterBase:
 
     To use in an instrument pipeline a subclass is required. Here's a simple, but complete example::
 
-        class InstParameters(ParameterBase):
-            def __init__(self, scratch, some_other_parameters):
-                super().__init__(scratch=scratch)
+        class InstParameters(ParameterBase)
+            def __init__(self, input_dataset_parameters, some_other_parameter):
+                super().__init__(input_dataset_parameters)
                 self._thing = self._some_function(some_other_parameters)
 
             @property
@@ -43,7 +34,7 @@ class ParameterBase:
                 return self._find_most_recent_past_value("some_parameter_name")
 
             @property
-            def complicated_parameter(self):
+            def complicate_parameter(self):
                 return self._some_complicated_parsing_function("complicated_parameter_name", another_argument)
 
 
@@ -64,16 +55,15 @@ class ParameterBase:
                     workflow_version=workflow_version,
                 )
 
-                self.parameters = InstParameters(scratch=self.scratch)  #<------ This is the important line
+                self.parameters = InstParameters(self.input_dataset_parameters)  #<------ This is the important line
 
-    ParameterBase needs the task scratch in order to read the parameters document written at input dataset
-    transfer.  Note that the first argument to the ConstantsSubclass will *always* be scratch, but additional
-    arguments can be passed if the subclass requires them.
+    Note that the first argument to the ConstantsSubclass with *always* be self.input_dataset_parameters, but
+    additional argument can be passed if the subclass requires them.
 
     Parameters
     ----------
-     scratch
-        The task scratch WorkflowFileSystem instance
+    input_dataset_parameters
+        The input parameters
 
     obs_ip_start_time
         A string containing the start date of the Observe IP task type frames. Must be in isoformat.
@@ -84,53 +74,25 @@ class ParameterBase:
 
     def __init__(
         self,
-        scratch: WorkflowFileSystem,
+        input_dataset_parameters: dict[str, list[InputDatasetParameterValue]],
         obs_ip_start_time: str | None = None,
         **kwargs,
     ):
-        self.scratch = scratch
-        input_dataset_parameter_model = self._get_parameters_doc_from_file()
-        input_dataset_parameters = {}
-        if input_dataset_parameter_model is not None:
-            input_dataset_parameters = {
-                p.parameter_name: p.parameter_values for p in input_dataset_parameter_model.doc_list
-            }
         self.input_dataset_parameters = input_dataset_parameters
-
         if obs_ip_start_time is not None:
             # Specifically `not None` because we want to error normally on badly formatted strings (including "").
             self._obs_ip_start_datetime = datetime.fromisoformat(obs_ip_start_time)
         else:
             logger.info(
                 "WARNING: "
-                "The task containing this parameters object did not provide an obs ip start time, "
-                "which really only makes sense for Parsing tasks."
+                "The task containing this parameters object did not provide an obs ip start time. "
+                "This really only makes sense for Parsing tasks."
             )
 
         for parent_class in self.__class__.__bases__:
             if hasattr(parent_class, "is_param_mixin"):
                 parent_class.__init__(self, **kwargs)
 
-    def _read_parameter_file(
-        self, tag: str, decoder: Callable[[Path], Any], **decoder_kwargs
-    ) -> Any:
-        """Read any file in the task scratch instance."""
-        paths = list(self.scratch.find_all(tags=tag))
-        if len(paths) == 0:
-            logger.info(f"WARNING: There is no parameter file for {tag = }")
-        if len(paths) == 1:
-            return decoder(paths[0], **decoder_kwargs)
-        if len(paths) > 1:
-            raise ValueError(f"There is more than one parameter file for {tag = }: {paths}")
-
-    def _get_parameters_doc_from_file(self) -> InputDatasetPartDocumentList:
-        """Get parameters doc saved at the TransferL0Data task."""
-        tag = Tag.input_dataset_parameters()
-        parameters_from_file = self._read_parameter_file(
-            tag=tag, decoder=basemodel_decoder, model=InputDatasetPartDocumentList
-        )
-        return parameters_from_file
-
     def _find_most_recent_past_value(
         self,
         parameter_name: str,
@@ -151,19 +113,20 @@ class ParameterBase:
             )
         return result
 
-    def _load_param_value_from_fits(
-        self, param_obj: InputDatasetFilePointer, hdu: int = 0
-    ) -> np.ndarray:
-        """Return the data associated with a tagged parameter file saved in FITS format."""
-        tag = param_obj.file_pointer.tag
-        param_value = self._read_parameter_file(tag=tag, decoder=fits_array_decoder, hdu=hdu)
-        return param_value
-
-    def _load_param_value_from_numpy_save(self, param_obj: InputDatasetFilePointer) -> np.ndarray:
-        """Return the data associated with a tagged parameter file saved in numpy format."""
-        tag = param_obj.file_pointer.tag
-        param_value = self._read_parameter_file(tag=tag, decoder=array_decoder)
-        return param_value
+    @staticmethod
+    def _load_param_value_from_fits(param_dict: dict, hdu: int = 0) -> np.ndarray:
+        """Load a numpy array from a parameter pointing to a FITS file."""
+        file_path = param_dict["param_path"]
+
+        hdul = fits.open(file_path)
+        return hdul[hdu].data
+
+    @staticmethod
+    def _load_param_value_from_numpy_save(param_dict: dict) -> np.ndarray:
+        """Return the data associated with a parameter file saved in numpy format."""
+        file_path = param_dict["param_path"]
+        result = np.load(file_path)
+        return result
 
 
 class _ParamMixinBase:

{dkist_processing_common-10.8.1rc1 → dkist_processing_common-10.8.3}/dkist_processing_common/parsers/quality.py

@@ -31,3 +31,4 @@ class L1QualityFitsAccess(L1FitsAccess):
         self.light_level: float = self.header["LIGHTLVL"]
         self.health_status: str = self.header["DSHEALTH"]
         self.ao_status: int = self.header.get("AO_LOCK", None)
+        self.num_out_of_bounds_ao_values: int = self.header.get("OOBSHIFT", None)

dkist_processing_common-10.8.3/dkist_processing_common/tasks/mixin/input_dataset.py

@@ -0,0 +1,166 @@
+"""Mixin for a WorkflowDataTaskBase subclass which implements input data set access functionality."""
+import json
+from dataclasses import dataclass
+from datetime import datetime
+from itertools import chain
+from pathlib import Path
+from typing import Any
+
+from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.tasks.base import tag_type_hint
+
+
+frames_part_type_hint = list[dict[str, str | list[str]]] | None
+
+
+@dataclass
+class InputDatasetParameterValue:
+    """Data structure for a de-serialized input dataset parameter value."""
+
+    parameter_value_id: int
+    parameter_value: Any = None
+    parameter_value_start_date: datetime | None = None
+
+
+@dataclass
+class InputDatasetObject:
+    """Data structure for a de-serialized input dataset frame."""
+
+    bucket: str
+    object_key: str
+
+
+class InputDatasetMixin:
+    """Mixin for WorkflowDataTaskBase that accesses downloaded input dataset part documents."""
+
+    def _input_dataset_part_document(self, tags: tag_type_hint):
+        """Get the input dataset document part and deserialize it."""
+        paths: list[Path] = list(self.read(tags=tags))
+        if not paths:
+            return
+        if len(paths) > 1:
+            raise ValueError(
+                f"There are more than one input dataset part documents to parse for {tags=}"
+            )
+        p = paths[0]
+        with p.open(mode="rb") as f:
+            return json.load(f)
+
+    @property
+    def input_dataset_observe_frames_part_document(self) -> frames_part_type_hint:
+        """Get the 'observe frames' part of the input dataset."""
+        return self._input_dataset_part_document(tags=Tag.input_dataset_observe_frames())
+
+    @property
+    def input_dataset_calibration_frames_part_document(self) -> frames_part_type_hint:
+        """Get the 'calibration frames' part of the input dataset."""
+        return self._input_dataset_part_document(tags=Tag.input_dataset_calibration_frames())
+
+    @property
+    def input_dataset_parameters_part_document(
+        self,
+    ) -> list[dict[str, str | list[dict[str, int | str]]]] | None:
+        """Get the 'parameters' part of the input dataset."""
+        return self._input_dataset_part_document(tags=Tag.input_dataset_parameters())
+
+    @property
+    def input_dataset_frames(self) -> list[InputDatasetObject]:
+        """Get the list of frames for this input dataset."""
+        result = []
+        observe_frames = self.input_dataset_observe_frames_part_document or []
+        calibration_frames = self.input_dataset_calibration_frames_part_document or []
+        for frame_set in chain(observe_frames, calibration_frames):
+            for key in frame_set.get("object_keys", list()):
+                result.append(InputDatasetObject(bucket=frame_set["bucket"], object_key=key))
+        return result
+
+    @property
+    def input_dataset_parameters(self) -> dict[str, list[InputDatasetParameterValue]]:
+        """Get the input dataset parameters."""
+        parameters = self.input_dataset_parameters_part_document or []
+        result = dict()
+        for p in parameters:
+            result.update(self._input_dataset_parse_parameter(p))
+        return result
+
+    @property
+    def input_dataset_parameter_objects(self) -> list[InputDatasetObject]:
+        """Parse the parameter object locations out of the set of all parameters."""
+        result = []
+        for value_list in self.input_dataset_parameters.values():
+            for value in value_list:
+                param_value = value.parameter_value
+                if isinstance(param_value, dict) and param_value.get("is_file", False):
+                    result.append(
+                        InputDatasetObject(
+                            bucket=param_value["bucket"], object_key=param_value["objectKey"]
+                        )
+                    )
+        return result
+
+    def _input_dataset_parse_parameter(
+        self, parameter: dict
+    ) -> dict[str, list[InputDatasetParameterValue]]:
+        name: str = parameter["parameterName"]
+        raw_values: list[dict] = parameter["parameterValues"]
+        values = self._input_dataset_parse_parameter_values(raw_values=raw_values)
+        return {name: values}
+
+    def _input_dataset_parse_parameter_values(
+        self, raw_values: list[dict[str, Any]]
+    ) -> list[InputDatasetParameterValue]:
+        values = list()
+        for v in raw_values:
+            parsed_value = InputDatasetParameterValue(parameter_value_id=v["parameterValueId"])
+            parsed_value.parameter_value = self._input_dataset_parse_parameter_value(
+                raw_parameter_value=v["parameterValue"]
+            )
+            if d := v.get("parameterValueStartDate"):
+                parsed_value.parameter_value_start_date = datetime.fromisoformat(d)
+            else:
+                parsed_value.parameter_value_start_date = datetime(1, 1, 1)
+            values.append(parsed_value)
+        return values
+
+    def _input_dataset_parse_parameter_value(self, raw_parameter_value: str) -> Any:
+        """Return the json decoding of the parameter value."""
+        return json.loads(raw_parameter_value, object_hook=self._decode_parameter_value)
+
+    def _decode_parameter_value(self, param_dict: dict):
+        """Decode a parameter value."""
+        if "__file__" in param_dict:
+            return self._convert_parameter_file_value_to_path(param_dict)
+        # Nothing to do here, so return control back to json.loads()
+        return param_dict
+
+    def _convert_parameter_file_value_to_path(self, param_dict: dict):
+        """
+        Decode a parameter file value.
+
+        Note: for parameters that are files, the string passed to json.loads() looks like this:
+        '{ "__file__":
+            {
+                "bucket": "data",
+                "objectKey": "parameters/parameter_name/uuid.dat"
+            }
+        }'
+
+        In this hook, we remove the outer __file__ dict and return the inner dict with the addition of
+        a key and value for the file path. Because the file path is dependent on the existence of a tag,
+        if no tag is found, None is returned for the path. This use case will occur when we construct
+        the list of files to be transferred and tagged by the TransferL0Data task.
+        """
+        file_dict = param_dict["__file__"]
+        object_key = file_dict["objectKey"]
+        parameter_tag = Tag.parameter(Path(object_key).name)
+        paths = list(self.read(tags=parameter_tag))
+        num_paths = len(paths)
+        if num_paths == 1:
+            param_path = paths[0]
+        elif num_paths == 0:
+            param_path = None
+        else:
+            raise ValueError(f"Found multiple paths for {parameter_tag = }.")
+        file_dict["param_path"] = param_path
+        file_dict["is_file"] = True
+        return file_dict

{dkist_processing_common-10.8.1rc1 → dkist_processing_common-10.8.3}/dkist_processing_common/tasks/mixin/metadata_store.py

@@ -210,19 +210,16 @@ class MetadataStoreMixin:
         self, part_type: Literal["observe_frames", "calibration_frames", "parameters"]
     ) -> InputDatasetPartResponse:
         """Get the input dataset part by input dataset part type name."""
-        part_types_found = set()
-        input_dataset_part = None
+        part_type_dict = {}
         parts = (
             self.metadata_store_input_dataset_recipe_run.recipeInstance.inputDataset.inputDatasetInputDatasetParts
         )
         for part in parts:
             part_type_name = part.inputDatasetPart.inputDatasetPartType.inputDatasetPartTypeName
-            if part_type_name in part_types_found:
+            if part_type_name in part_type_dict.keys():
                 raise ValueError(f"Multiple input dataset parts found for {part_type_name=}.")
-            part_types_found.add(part_type_name)
-            if part_type_name == part_type:
-                input_dataset_part = part.inputDatasetPart
-        return input_dataset_part
+            part_type_dict[part_type_name] = part.inputDatasetPart
+        return part_type_dict.get(part_type)
 
     @property
     def metadata_store_input_dataset_observe_frames(self) -> InputDatasetPartResponse:

{dkist_processing_common-10.8.1rc1 → dkist_processing_common-10.8.3}/dkist_processing_common/tasks/mixin/quality/_metrics.py

@@ -20,6 +20,7 @@ from dkist_processing_pac.fitter.fitting_core import compare_I
 from dkist_processing_pac.fitter.polcal_fitter import PolcalFitter
 from pandas import DataFrame
 
+from dkist_processing_common.models.fried_parameter import r0_valid
 from dkist_processing_common.models.metric_code import MetricCode
 from dkist_processing_common.models.quality import EfficiencyHistograms
 from dkist_processing_common.models.quality import ModulationMatrixHistograms
@@ -203,7 +204,7 @@ class _SimplePlotQualityMixin:
         return warnings
 
     def quality_store_ao_status_and_fried_parameter(
-        self, datetimes: list[str], values: list[list[bool, float]]
+        self, datetimes: list[str], values: list[list[bool | float]]
     ):
         """
         Collect and store datetime / value pairs for the boolean AO status and Fried parameter.
@@ -213,23 +214,27 @@ class _SimplePlotQualityMixin:
         Because of how L1Metric.has_metric works, empty lists will not be passed to this method.
         However, because of how L1Metric.store_metric works, one or both values can be None.
         """
-        ao_values = [value[0] for value in values]
-        ao_not_none = [ao for ao in ao_values if ao is not None]
+        ao_lock_values = [value[0] for value in values]
+        ao_not_none = [ao for ao in ao_lock_values if ao is not None]
         if len(ao_not_none) != 0:
             self._record_values(values=ao_not_none, tags=Tag.quality(MetricCode.ao_status))
         fried_values = [value[1] for value in values]
-        fried_values_not_none_ao_locked = [
-            r0 for ao, r0 in zip(ao_values, fried_values) if (ao is True) and (r0 is not None)
-        ]
-        datetimes_fried_not_none_ao_locked = [
-            dt
-            for dt, ao, r0 in zip(datetimes, ao_values, fried_values)
-            if (ao is True) and (r0 is not None)
-        ]
-        if len(fried_values_not_none_ao_locked) != 0:
+        ao_oob_values = [value[2] for value in values]
+        fried_values_to_plot = []
+        datetimes_to_plot = []
+        # For each set of input data, check if the r0 is considered valid based on all data
+        for i in range(len(fried_values)):
+            if r0_valid(
+                r0=fried_values[i],
+                ao_lock=ao_lock_values[i],
+                num_out_of_bounds_ao_values=ao_oob_values[i],
+            ):
+                fried_values_to_plot.append(fried_values[i])
+                datetimes_to_plot.append(datetimes[i])
+        if len(fried_values_to_plot) != 0:
             self._record_2d_plot_values(
-                x_values=datetimes_fried_not_none_ao_locked,
-                y_values=fried_values_not_none_ao_locked,
+                x_values=datetimes_to_plot,
+                y_values=fried_values_to_plot,
                 tags=Tag.quality(MetricCode.fried_parameter),
             )
 

{dkist_processing_common-10.8.1rc1 → dkist_processing_common-10.8.3}/dkist_processing_common/tasks/quality_metrics.py

@@ -296,7 +296,7 @@ class QualityL1Metrics(WorkflowTaskBase, QualityMixin):
             L1Metric(storage_method=self.quality_store_health_status, value_source="health_status"),
             L1Metric(
                 storage_method=self.quality_store_ao_status_and_fried_parameter,
-                value_source=["ao_status", "fried_parameter"],
+                value_source=["ao_status", "fried_parameter", "num_out_of_bounds_ao_values"],
             ),
         ]