dkist-processing-common 13.0.0rc2__tar.gz → 13.0.1__tar.gz

{dkist_processing_common-13.0.0rc2 → dkist_processing_common-13.0.1}/CHANGELOG.rst

@@ -1,3 +1,40 @@
+v13.0.1 (2026-04-07)
+====================
+
+Misc
+----
+
+- Use dkist-fits-specifications v4.24.0, which changes some Cryo-NIRSP dataset extra keys to be not instrument required. (`#324 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/324>`__)
+
+
+v13.0.0 (2026-04-03)
+====================
+
+Bugfixes
+--------
+
+- Update `*CadenceBuds` to correctly account for polarimetric data. Previously, the potentially different cadence
+  between modulation states was not considered when computing the cadence of L0 observations. (`#310 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/310>`__)
+
+
+Misc
+----
+
+- *All* children of `L0FitsAccess` are now required to define the `modulator_state` attribute. If an instrument is not
+  polarimetric, setting this attribute to -1 is recommended. (`#310 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/310>`__)
+
+
+v12.10.0 (2026-04-02)
+=====================
+
+Features
+--------
+
+- Added capabilities to simplify the way that quality metrics get produced.
+  The central feature is the `QualityMetric` `BaseModel` which can be used to fully define any quality metric. (`#313 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/313>`__)
+- Simplify the production of L0 quality metrics. (`#313 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/313>`__)
+
+
 v12.9.0 (2026-04-01)
 ====================
 

{dkist_processing_common-13.0.0rc2 → dkist_processing_common-13.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 13.0.0rc2
+Version: 13.0.1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -15,7 +15,7 @@ Requires-Python: >=3.13
 Description-Content-Type: text/x-rst
 Requires-Dist: asdf<4.0.0,>=3.5.0
 Requires-Dist: astropy>=7.0.0
-Requires-Dist: dkist-fits-specifications<5.0,>=4.23.0
+Requires-Dist: dkist-fits-specifications<5.0,>=4.24.0
 Requires-Dist: dkist-header-validator<6.0,>=5.3.0
 Requires-Dist: dkist-processing-core==7.1.0
 Requires-Dist: dkist-processing-pac<4.0,>=3.1

dkist_processing_common-13.0.1/dkist_processing_common/models/quality.py

@@ -0,0 +1,351 @@
+"""Support classes used to create a quality report."""
+
+from datetime import datetime
+from typing import Annotated
+from typing import Any
+
+from pydantic import BaseModel
+from pydantic import BeforeValidator
+from pydantic import ConfigDict
+from pydantic import Field
+from pydantic import FiniteFloat
+from pydantic import PlainSerializer
+from pydantic import field_validator
+from pydantic import model_serializer
+from pydantic import model_validator
+from pydantic.alias_generators import to_camel
+from pydantic_core.core_schema import SerializationInfo
+from pydantic_core.core_schema import ValidationInfo
+
+###########################################################
+# Old Quality Models                                      #
+#                                                         #
+# These will gradually be replaced by                    #
+# New Quality Models (below)                              #
+###########################################################
+
+
+class Plot2D(BaseModel):
+    """Support class use to hold the data for creating a 2D plot in the quality report."""
+
+    xlabel: str
+    ylabel: str
+    series_data: dict[str, list[list[Any]]]
+    series_name: str | None = None
+    ylabel_horizontal: bool = False
+    ylim: tuple[float, float] | None = None
+    plot_kwargs: dict[str, dict[str, Any]] = Field(default_factory=dict)
+    sort_series: bool = True
+
+
+class VerticalMultiPanePlot2D(BaseModel):
+    """
+    Support class to hold a multi-pane plot with plots stacked vertically.
+
+    This type of metric is really geared towards plots that share an X axis and have no gap between them. If you just
+    want two separate plots it's probably better to use a list of `Plot2D` objects.
+    """
+
+    top_to_bottom_plot_list: list[Plot2D]
+    match_x_axes: bool = True
+    no_gap: bool = True
+    top_to_bottom_height_ratios: list[float] | None = None
+
+    @field_validator("top_to_bottom_height_ratios")
+    @classmethod
+    def ensure_same_number_of_height_ratios_and_plots(
+        cls, height_ratios: list[float] | None, info: ValidationInfo
+    ) -> list[float]:
+        """
+        Make sure that the number of height ratios is the same as the number of plots.
+
+        Also populates default, same-size ratios if no ratios were given.
+        """
+        try:
+            plot_list = info.data["top_to_bottom_plot_list"]
+        except KeyError:
+            # The plot list didn't validate for some reason. We're about to error anyway.
+            return [1.0]
+
+        num_plots = len(plot_list)
+        if height_ratios is None:
+            return [1.0] * num_plots
+
+        if len(height_ratios) != num_plots:
+            raise ValueError(
+                f"The number of items in `top_to_bottom_height_ratios` list ({len(height_ratios)}) is not "
+                f"the same as the number of plots ({num_plots})"
+            )
+
+        return height_ratios
+
+
+class SimpleTable(BaseModel):
+    """Support class to hold a simple table to be inserted into the quality report."""
+
+    rows: list[list[Any]]
+    header_row: bool = True
+    header_column: bool = False
+
+
+class ModulationMatrixHistograms(BaseModel):
+    """Support class for holding the big ol' grid of histograms that represent the modulation matrix fits."""
+
+    modmat_list: list[list[list[float]]]
+
+
+class EfficiencyHistograms(BaseModel):
+    """Support class for holding 4 histograms that correspond to efficiencies of the 4 stokes components."""
+
+    efficiency_list: list[list[float]]
+
+
+class PlotHistogram(BaseModel):
+    """Support class to hold 1D data for plotting a histogram."""
+
+    xlabel: str
+    series_data: dict[str, list[float]]
+    series_name: str | None = None
+    vertical_lines: dict[str, float] | None
+
+
+class PlotRaincloud(BaseModel):
+    """Support class to hold data series for fancy-ass violin plots."""
+
+    xlabel: str
+    ylabel: str
+    categorical_column_name: str
+    distribution_column_name: str
+    dataframe_json: str
+    hue_column_name: str | None
+    ylabel_horizontal: bool | None
+
+
+class ReportMetric(BaseModel):
+    """A Quality Report is made up of a list of metrics with the schema defined by this class."""
+
+    name: str
+    description: str
+    metric_code: str
+    facet: str | None = None
+    statement: str | list[str] | None = None
+    plot_data: Plot2D | list[Plot2D] | None = None
+    multi_plot_data: VerticalMultiPanePlot2D | None = None
+    histogram_data: PlotHistogram | list[PlotHistogram] | None = None
+    table_data: SimpleTable | list[SimpleTable] | None = None
+    modmat_data: ModulationMatrixHistograms | None = None
+    efficiency_data: EfficiencyHistograms | None = None
+    raincloud_data: PlotRaincloud | None = None
+    warnings: list[str] | None = None
+
+
+###########################################################
+# New Quality Models                                      #
+#                                                         #
+# These define the interface with dkist-quality           #
+# These will eventually get moved to dkist-quality        #
+###########################################################
+
+
+class QualityModel(BaseModel):
+    """BaseModel for quality data."""
+
+    model_config = ConfigDict(
+        # only accept snake_case field names
+        validate_by_name=True,
+        # re-validate the field if its value gets changed after creation
+        validate_assignment=True,
+    )
+
+
+def datetime_or_iso_dict_input(v: Any) -> Any:
+    """
+    Extend pydantic `datetime` capability.
+
+    Adds the ability to accept an 'iso dict' of the {"iso_date": <value>} form
+    """
+    # If appropriate, extract value from {"iso_date": <value>} before passing to the normal datetime handler
+    if isinstance(v, dict) and "iso_date" in v:
+        return v["iso_date"]
+    return v
+
+
+# NOTE:  This is a temporary data type that can be eliminated once all metric conversion is complete
+DatetimeIsoDict = Annotated[
+    # this is a datetime within the pydantic BaseModel
+    datetime,
+    # accept 'iso dict' input in addition to normal datetime inputs
+    BeforeValidator(datetime_or_iso_dict_input),
+    # always output the datetime as an 'iso dict'
+    PlainSerializer(lambda dt: {"iso_date": dt.isoformat()}, return_type=dict[str, str]),
+]
+
+
+class TimeSeriesData(QualityModel):
+    """
+    Time Series Data.
+
+    The context, e.g. task_type or modstate, is maintained outside of this structure.
+    """
+
+    # by default, a tuple accepts a list as input
+    # the use of tuples is intentional because list mutations do not trigger validation
+    # the impact is that TimeSeriesData should not be instantiated until
+    # after the tuple input has been fully accumulated
+    # when `mode="json"`, the output of a tuple results in a list
+    # when `mode="python"`, the output of a tuple results in a tuple
+    x_values: tuple[DatetimeIsoDict, ...]
+    # NOTE:  FiniteFloat will fail fast for `nan` and `inf`
+    y_values: tuple[FiniteFloat, ...]
+
+    @model_validator(mode="after")
+    def tuple_validation(self):
+        """Validate tuple lengths."""
+        axes_are_different_lengths = len(self.x_values) != len(self.y_values)
+        if axes_are_different_lengths:
+            raise ValueError(
+                f"Cannot store TimeSeriesData with different length axes. "
+                f"{len(self.x_values)=}, {len(self.y_values)=}"
+            )
+        # Can't have this AND default to empty tuple
+        axes_are_zero_length = not self.x_values or not self.y_values
+        if axes_are_zero_length:
+            raise ValueError(
+                f"Cannot store TimeSeriesData with 0 length axes. "
+                f"{len(self.x_values)=}, {len(self.y_values)=}"
+            )
+        return self
+
+    @model_serializer(mode="wrap")
+    def custom_dump(self, handler, info: SerializationInfo):
+        """Serialize output for QRM compatibility."""
+        data = handler(self)
+        # change structure if mode="json" for QRM compatibility
+        if info.mode == "json":
+            # dump as list[list[Any]] with DatetimeIsoDict format for x_values
+            return [list(data["x_values"]), list(data["y_values"])]
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def handle_json(cls, input_value: Any):
+        """
+        Transform json input from list[list[Any]] to dict[Any, Any].
+
+        the dict is expected to be dict[str, tuple[Any, ...]], which gets validated by normal BaseModel configurations.
+        """
+        if isinstance(input_value, dict):
+            return input_value
+
+        if (
+            isinstance(input_value, (list, tuple))
+            and len(input_value) == 2
+            and isinstance(input_value[0], (list, tuple))
+            and isinstance(input_value[1], (list, tuple))
+        ):
+            return {"x_values": input_value[0], "y_values": input_value[1]}
+
+        return input_value
+
+
+class XYData(QualityModel):
+    """One or more time series for a single x-y plot."""
+
+    xlabel: str
+    ylabel: str
+    # the key is the name assigned to the series, e.g. within the legend
+    # the key cannot be None, because `null` is not a valid json key
+    series_data: dict[str, TimeSeriesData] = Field(default_factory=dict)
+    # TODO FUTURE - convert `series_name` to `legend_title` to better align with matplotlib terminology
+    series_name: str | None = None
+    ylabel_horizontal: bool | None = None
+    ylim: tuple[float | None, float | None] | None = None
+    plot_kwargs: dict[str, dict[str, Any]] = Field(default_factory=dict)
+    sort_series: bool | None = True
+
+
+class TableData(QualityModel):
+    """Tabular Data."""
+
+    # by default, a tuple accepts a list as input
+    # the use of tuples is intentional because list mutations do not trigger validation
+    # the impact is that TableData should not be instantiated until
+    # after the tuple input has been fully accumulated
+    # when `mode="json"`, the output of a tuple results in a list
+    # when `mode="python"`, the output of a tuple results in a tuple
+    # Note: numeric values are represented as strings for table data
+    rows: tuple[tuple[str, ...], ...]
+    header_row: bool | None = True
+    header_col: bool | None = False
+
+    @model_validator(mode="after")
+    def tuple_validation(self):
+        """Validate tuple lengths."""
+        row_count = len(self.rows)
+        if row_count == 0:
+            raise ValueError(f"Invalid table. No rows.")
+        header_column_count = len(self.rows[0])
+        if header_column_count == 0:
+            raise ValueError(f"Invalid table. No columns.")
+        row_column_counts = [len(row) for row in self.rows[1:]]
+        column_count_mismatch = any(
+            col_count != header_column_count for col_count in row_column_counts
+        )
+        if column_count_mismatch:
+            raise ValueError(
+                f"Mismatch in column count. {header_column_count=}, {row_column_counts=}"
+            )
+        return self
+
+
+class MultiPlotData(QualityModel):
+    """Multiple Plot Data."""
+
+    # TBD
+    pass
+
+
+class HistogramData(QualityModel):
+    """Histogram Data."""
+
+    # TBD
+    pass
+
+
+class ModMatData(QualityModel):
+    """Modulation Matrix Data."""
+
+    # TBD
+    pass
+
+
+class RaincloudData(QualityModel):
+    """Raincloud Data."""
+
+    # TBD
+    pass
+
+
+class EfficiencyData(QualityModel):
+    """Efficiency Data."""
+
+    # TBD
+    pass
+
+
+class QualityMetric(QualityModel):
+    """Quality Metric."""
+
+    name: str
+    description: str
+    metric_code: str
+    facet: str | None = None
+    statement: list[str] | None = None
+    plot_data: list[XYData] | None = None
+    multi_plot_data: list[MultiPlotData] | None = None
+    histogram_data: list[HistogramData] | None = None
+    table_data: list[TableData] | None = None
+    modmad_data: list[ModMatData] | None = None
+    raincloud_data: list[RaincloudData] | None = None
+    efficiency_data: list[EfficiencyData] | None = None
+    warnings: list[str] | None = None

{dkist_processing_common-13.0.0rc2 → dkist_processing_common-13.0.1}/dkist_processing_common/tasks/l1_output_data.py

@@ -5,11 +5,13 @@ from abc import ABC
 from pathlib import Path
 from typing import Iterable
 
+from dkist_processing_common.codecs.basemodel import basemodel_decoder
 from dkist_processing_common.codecs.quality import quality_data_encoder
 from dkist_processing_common.models.message import CatalogFrameMessage
 from dkist_processing_common.models.message import CatalogFrameMessageBody
 from dkist_processing_common.models.message import CatalogObjectMessage
 from dkist_processing_common.models.message import CatalogObjectMessageBody
+from dkist_processing_common.models.quality import QualityMetric
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.mixin.globus import GlobusMixin
 from dkist_processing_common.tasks.mixin.interservice_bus import InterserviceBusMixin
@@ -146,8 +148,13 @@ class AssembleQualityData(L1OutputDataBase, QualityMixin):
 
     def run(self):
         """Run method for the task."""
-        with self.telemetry_span("Assembling quality data"):
-            quality_data = self.quality_assemble_data(polcal_label_list=self.polcal_label_list)
+        # this is the new way, which will eventually supplant the old `quality_assemble_data`
+        with self.telemetry_span("Assembling generic quality data"):
+            quality_data = self.assemble_quality_data()
+
+        # this is the old way, which is being replaced by the new `assemble_quality_data`
+        with self.telemetry_span("Assembling quality data by Metric"):
+            quality_data += self.quality_assemble_data(polcal_label_list=self.polcal_label_list)
 
         with self.telemetry_span(
             f"Saving quality data with {len(quality_data)} metrics to the file system"
@@ -159,6 +166,16 @@ class AssembleQualityData(L1OutputDataBase, QualityMixin):
                 relative_path=f"{self.constants.dataset_id}_quality_data.json",
             )
 
+    def assemble_quality_data(self) -> list[dict]:
+        """Assemble Quality Data."""
+        metrics = []
+        tags = [Tag.quality("GENERIC")]
+        for metric in self.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric):
+            # `metrics` needs to be list[dict] for compatibility with existing `quality_assemble_data`
+            # once all metric conversion is complete, `metrics` should be list[QualityMetric] instead
+            metrics.append(metric.model_dump(mode="json"))
+        return metrics
+
 
 class SubmitDatasetMetadata(L1OutputDataBase):
     """

{dkist_processing_common-13.0.0rc2 → dkist_processing_common-13.0.1}/dkist_processing_common/tasks/mixin/quality/_base.py

@@ -28,7 +28,6 @@ class QualityMixin(
         """Assemble the quality data by checking for the existence of each metric."""
         report = []
         report += self.quality_task_independent_metrics()
-        report += self.quality_task_dependent_metrics()
 
         polcal_labels = polcal_label_list or []
         report += self.quality_polcal_metrics(polcal_labels)
@@ -43,15 +42,6 @@ class QualityMixin(
                 result.append(metric_func())
         return result
 
-    def quality_task_dependent_metrics(self) -> list[dict]:
-        """Encapsulate task dependent metric parsing."""
-        result = []
-        for metric_name, metric_func in self.quality_metrics_task_dependence.items():
-            for task_type in self.quality_task_types:
-                if self._quality_metric_exists(metric_name=metric_name, task_type=task_type):
-                    result.append(metric_func(task_type=task_type))
-        return result
-
     def quality_polcal_metrics(self, label_list: list) -> list[dict]:
         """Encapsulate polcal metric parsing."""
         result = []
@@ -82,22 +72,12 @@ class QualityMixin(
             "SENSITIVITY": self.quality_build_sensitivity,
             "HEALTH_STATUS": self.quality_build_health_status,
             "TASK_TYPES": self.quality_build_task_type_counts,
-            "DATASET_AVERAGE": self.quality_build_dataset_average,
-            "DATASET_RMS": self.quality_build_dataset_rms,
             "HISTORICAL": self.quality_build_historical,
             "AO_STATUS": self.quality_build_ao_status,
             "RANGE": self.quality_build_range,
             "WAVECAL_FIT": self.quality_build_wavecal_results,
         }
 
-    @property
-    def quality_metrics_task_dependence(self) -> dict:
-        """Return a dict of quality metrics which are dependent on the task."""
-        return {
-            "FRAME_AVERAGE": self.quality_build_frame_average,
-            "FRAME_RMS": self.quality_build_frame_rms,
-        }
-
     @property
     def quality_metrics_polcal(self) -> dict:
         """Return a dict of polcal quality metrics."""