dkist-processing-common 13.0.4__tar.gz → 13.0.5__tar.gz

{dkist_processing_common-13.0.4 → dkist_processing_common-13.0.5}/CHANGELOG.rst

@@ -1,3 +1,12 @@
+v13.0.5 (2026-04-20)
+====================
+
+Features
+--------
+
+- Remove 4 unused quality metric codes: RANGE, NOISE, SENSITIVITY, HISTORICAL. (`#329 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/329>`__)
+
+
 v13.0.4 (2026-04-16)
 ====================
 

{dkist_processing_common-13.0.4 → dkist_processing_common-13.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 13.0.4
+Version: 13.0.5
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-13.0.4 → dkist_processing_common-13.0.5}/dkist_processing_common/models/metric_code.py

@@ -13,15 +13,11 @@ class MetricCode(StrEnum):
     frame_rms = "FRAME_RMS"
     fried_parameter = "FRIED_PARAMETER"
     health_status = "HEALTH_STATUS"
-    historical = "HISTORICAL"
     light_level = "LIGHT_LEVEL"
-    noise = "NOISE"
     polcal_constant_par_vals = "POLCAL_CONSTANT_PAR_VALS"
     polcal_efficiency = "POLCAL_EFFICIENCY"
     polcal_fit_residuals = "POLCAL_FIT_RESIDUALS"
     polcal_global_par_vals = "POLCAL_GLOBAL_PAR_VALS"
     polcal_local_par_vals = "POLCAL_LOCAL_PAR_VALS"
-    range = "RANGE"
-    sensitivity = "SENSITIVITY"
     task_types = "TASK_TYPES"
     wavecal_fit = "WAVECAL_FIT"

{dkist_processing_common-13.0.4 → dkist_processing_common-13.0.5}/dkist_processing_common/tasks/mixin/quality/_base.py

@@ -10,13 +10,11 @@ from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_common.tasks.mixin.quality._metrics import _PolcalQualityMixin
 from dkist_processing_common.tasks.mixin.quality._metrics import _SimplePlotQualityMixin
-from dkist_processing_common.tasks.mixin.quality._metrics import _SimpleQualityMixin
 from dkist_processing_common.tasks.mixin.quality._metrics import _TableQualityMixin
 from dkist_processing_common.tasks.mixin.quality._metrics import _WavecalQualityMixin
 
 
 class QualityMixin(
-    _SimpleQualityMixin,
     _SimplePlotQualityMixin,
     _TableQualityMixin,
     _PolcalQualityMixin,
@@ -66,11 +64,7 @@ class QualityMixin(
     def quality_metrics_no_task_dependence(self) -> dict:
         """Return a dict of the quality metrics with no task dependence."""
         return {
-            "NOISE": self.quality_build_noise,
-            "SENSITIVITY": self.quality_build_sensitivity,
             "TASK_TYPES": self.quality_build_task_type_counts,
-            "HISTORICAL": self.quality_build_historical,
-            "RANGE": self.quality_build_range,
             "WAVECAL_FIT": self.quality_build_wavecal_results,
         }
 
@@ -106,77 +100,3 @@ class QualityMixin(
         self.write(
             data=values, tags=tags, encoder=json_encoder, allow_nan=False, cls=QualityValueEncoder
         )
-
-    @staticmethod
-    def avg_noise(data) -> float:
-        """Estimate the average noise in the image."""
-        if len(data.shape) == 2:  # 2D data
-            corner_square_length = int(data.shape[0] * 0.2)  # 1/5th of x dimension of array
-            corner_square_height = int(data.shape[1] * 0.2)  # 1/5th of y dimension of array
-
-            square_1 = data[0:corner_square_length, 0:corner_square_height]  # top left
-
-            square_2 = data[-corner_square_length:, 0:corner_square_height]  # top right
-
-            square_3 = data[0:corner_square_length, -corner_square_height:]  # bottom left
-
-            square_4 = data[-corner_square_length:, -corner_square_height:]  # bottom right
-
-            return np.nanmean(
-                [
-                    np.nanstd(square_1),
-                    np.nanstd(square_2),
-                    np.nanstd(square_3),
-                    np.nanstd(square_4),
-                ]
-            )
-
-        if len(data.shape) == 3:  # 3D data
-            corner_cube_length = int(data.shape[0] * 0.2)  # 1/5th of x dimension of array
-            corner_cube_height = int(data.shape[1] * 0.2)  # 1/5th of y dimension of array
-            corner_cube_width = int(data.shape[2] * 0.2)  # 1/5th of z dimension of array
-
-            cube_1 = data[
-                0:corner_cube_length, 0:corner_cube_height, 0:corner_cube_width
-            ]  # top left front
-
-            cube_2 = data[
-                0:corner_cube_length, 0:corner_cube_height, -corner_cube_width:
-            ]  # top left back
-
-            cube_3 = data[
-                -corner_cube_length:, 0:corner_cube_height, 0:corner_cube_width
-            ]  # top right front
-
-            cube_4 = data[
-                -corner_cube_length:, 0:corner_cube_height, -corner_cube_width:
-            ]  # top right back
-
-            cube_5 = data[
-                0:corner_cube_length, -corner_cube_height:, 0:corner_cube_width
-            ]  # bottom left front
-
-            cube_6 = data[
-                0:corner_cube_length, -corner_cube_height:, -corner_cube_width:
-            ]  # bottom left back
-
-            cube_7 = data[
-                -corner_cube_length:, -corner_cube_height:, 0:corner_cube_width
-            ]  # bottom right front
-
-            cube_8 = data[
-                -corner_cube_length:, -corner_cube_height:, -corner_cube_width:
-            ]  # bottom right back
-
-            return np.nanmean(
-                [
-                    np.nanstd(cube_1),
-                    np.nanstd(cube_2),
-                    np.nanstd(cube_3),
-                    np.nanstd(cube_4),
-                    np.nanstd(cube_5),
-                    np.nanstd(cube_6),
-                    np.nanstd(cube_7),
-                    np.nanstd(cube_8),
-                ]
-            )

{dkist_processing_common-13.0.4 → dkist_processing_common-13.0.5}/dkist_processing_common/tasks/mixin/quality/_metrics.py

@@ -39,133 +39,9 @@ from dkist_processing_common.models.tags import Tag
 logger = logging.getLogger(__name__)
 
 
-class _SimpleQualityMixin:
-
-    def quality_store_range(self, name: str, warnings: list[str]):
-        """
-        Insert range checking warnings into the schema used to record quality info.
-
-        Parameters
-        ----------
-        name: name of the parameter / measurement for which range was out of bounds
-        warnings: list of warnings to be entered into the quality report
-        """
-        data = {"name": name, "warnings": warnings}
-        self._record_values(values=data, tags=Tag.quality(MetricCode.range))
-
-    def quality_build_range(self) -> dict:
-        """Build range data schema from stored data."""
-        warnings = []
-        # Loop over files that contain data for this metric
-        for path in self.read(tags=Tag.quality(MetricCode.range)):
-            with path.open() as f:
-                data = json.load(f)
-                for warning in data["warnings"]:
-                    warnings.append(warning)
-
-        return ReportMetric(
-            name="Range checks",
-            description="This metric is checking that certain input and calculated parameters "
-            "fall within a valid data range. If no parameters are listed here, all "
-            "pipeline parameters were measured to be in range",
-            metric_code=MetricCode.range,
-            warnings=self._format_warnings(warnings),
-        ).model_dump()
-
-
 class _SimplePlotQualityMixin:
     """Mixin containing metrics that present as simple x/y plots."""
 
-    @staticmethod
-    def _create_2d_plot_with_datetime_metric(
-        name: str,
-        description: str,
-        metric_code: str,
-        xlabel: str,
-        ylabel: str,
-        series_data: dict[str, list[list[Any]]],
-        series_name: str | None = None,
-        ylabel_horizontal: bool = False,
-        ylim: tuple[float, float] | None = None,
-        statement: str | None = None,
-        warnings: list[str] | None = None,
-        facet: str | None = None,
-    ) -> dict:
-        for k, v in series_data.items():
-            # Convert datetime strings to datetime objects
-            series_data[k][0] = [datetime.fromisoformat(i) for i in v[0]]
-            # Sort the lists to make sure they are in ascending time order
-            series_data[k][0], series_data[k][1] = (list(t) for t in zip(*sorted(zip(v[0], v[1]))))
-        plot_data = Plot2D(
-            series_data=series_data,
-            xlabel=xlabel,
-            ylabel=ylabel,
-            series_name=series_name,
-            ylabel_horizontal=ylabel_horizontal,
-            ylim=ylim,
-        )
-        metric = ReportMetric(
-            name=name,
-            description=description,
-            metric_code=metric_code,
-            facet=facet,
-            statement=statement,
-            plot_data=plot_data,
-            warnings=warnings,
-        )
-        return metric.model_dump()
-
-    def _record_2d_plot_values(
-        self,
-        x_values: list[str],
-        y_values: list[float],
-        tags: Iterable[str] | str,
-        series_name: str = "",
-        task_type: str | None = None,
-    ):
-        """
-        Encode values for a 2d plot type metric and store as a file.
-
-        Parameters
-        ----------
-        x_values: values to apply to the x axis of a 2d plot
-        y_values: values to apply to the y axis of a 2d plot
-        tags: list of tags relating to the specific quality parameter being stored
-        series_name: name of the series if this is part of a multi series plot metric
-        task_type: type of data to be used - dark, gain, etc
-        """
-        if isinstance(tags, str):
-            tags = [tags]
-        axis_are_different_lengths = len(x_values) != len(y_values)
-        axis_are_zero_length = not x_values or not y_values
-        if axis_are_different_lengths or axis_are_zero_length:
-            raise ValueError(
-                f"Cannot store 2D plot values with 0 length or different length axis. "
-                f"{len(x_values)=}, {len(y_values)=}"
-            )
-        data = {"x_values": x_values, "y_values": y_values, "series_name": series_name}
-        if task_type:
-            tags.append(Tag.quality_task(quality_task_type=task_type))
-        self._record_values(values=data, tags=tags)
-
-    def _load_2d_plot_values(self, tags: str | list[str], task_type: str | None = None):
-        """Load all quality files for a given tag and return the merged datetimes and values."""
-        if isinstance(tags, str):
-            tags = [tags]
-        if task_type:
-            tags.append(Tag.quality_task(quality_task_type=task_type))
-        all_plot_data = defaultdict(list)
-        for path in self.read(tags=tags):
-            with path.open() as f:
-                data = json.load(f)
-                series_name = str(data["series_name"])
-                if series_name in all_plot_data.keys():
-                    all_plot_data[series_name][0].extend(data["x_values"])
-                    all_plot_data[series_name][1].extend(data["y_values"])
-                else:
-                    all_plot_data[series_name] = [data["x_values"], data["y_values"]]
-        return all_plot_data
-
     @staticmethod
     def _find_iqr_outliers(datetimes: Sequence[datetime], values: Sequence[float]) -> list[str]:
         """
@@ -190,57 +66,6 @@ class _SimplePlotQualityMixin:
                 )
         return warnings
 
-    def quality_store_noise(self, datetimes: list[str], values: list[float], stokes: str = "I"):
-        """Collect and store datetime / value pairs for the noise data."""
-        self._record_2d_plot_values(
-            x_values=datetimes,
-            y_values=values,
-            series_name=stokes,
-            tags=[Tag.quality(MetricCode.noise), Tag.stokes(stokes)],
-        )
-
-    def quality_build_noise(self) -> dict:
-        """Build noise schema from stored data."""
-        series_data = self._load_2d_plot_values(tags=[Tag.quality(MetricCode.noise)])
-        return self._create_2d_plot_with_datetime_metric(
-            name=f"Noise Estimation",
-            description="Estimate of the noise in L1 frames. Noise is computed as the average of the stddev of "
-            "boxes/cubes that extend 1/5 from the edge of the images on all sides. "
-            "One measurement taken per L1 frame.",
-            metric_code=MetricCode.noise,
-            xlabel="Time",
-            ylabel="Noise (adu)",
-            series_data=series_data,
-            warnings=None,
-        )
-
-    def quality_store_sensitivity(
-        self, stokes: Literal["I", "Q", "U", "V"], datetimes: list[str], values: list[float]
-    ):
-        """Collect and store datetime / value pairs for the polarimetric noise data."""
-        self._record_2d_plot_values(
-            x_values=datetimes,
-            y_values=values,
-            series_name=stokes,
-            tags=[Tag.quality(MetricCode.sensitivity), Tag.stokes(stokes)],
-        )
-
-    def quality_build_sensitivity(self) -> dict:
-        """Build polarimetric noise schema from stored data."""
-        series_data = self._load_2d_plot_values(tags=[Tag.quality(MetricCode.sensitivity)])
-        return self._create_2d_plot_with_datetime_metric(
-            name=f"Sensitivity",
-            description=f"Sensitivity is defined as the stddev of a particular Stokes parameter divided by the signal in "
-            f"Stokes I (computed as a median over the whole frame). One measurement is shown per map scan.",
-            metric_code=MetricCode.sensitivity,
-            xlabel="Time",
-            ylabel=r"$\frac{\sigma_X}{\mathrm{med}(I)}$",
-            ylabel_horizontal=True,
-            series_data=series_data,
-            series_name="Stokes Parameter",
-            warnings=None,
-        )
-
 
 class _TableQualityMixin:
     """Mixing for metrics that present as tables."""
@@ -320,43 +145,6 @@ class _TableQualityMixin:
             warnings=self._format_warnings(warnings),
         )
 
-    def quality_store_historical(self, name: str, value: Any, warning: str | None = None):
-        """
-        Insert historical data into the schema used to record quality info.
-
-        Parameters
-        ----------
-        name: name of the parameter / measurement to be recorded
-        value: value of the parameter / measurement to be recorded
-        warning: warning to be entered into the quality report
-        """
-        data = {"name": name, "value": value, "warnings": warning}
-        self._record_values(values=data, tags=Tag.quality(MetricCode.historical))
-
-    def quality_build_historical(self) -> dict:
-        """Build historical data schema from stored data."""
-        table_data = []
-        warnings = []
-        # Loop over files that contain data for this metric
-        for path in self.read(tags=Tag.quality(MetricCode.historical)):
-            with path.open() as f:
-                data = json.load(f)
-                table_data.append([data["name"], data["value"]])
-                if data["warnings"] is not None:
-                    warnings.append(data["warnings"])
-
-        # Add header row
-        table_data.insert(0, ["Metric", "Value"])
-        return self._create_table_metric(
-            name="Historical Comparisons",
-            description="Over time, the data center will be comparing some of the above quality "
-            "metrics and other parameters derived from file headers to see how the "
-            "DKIST instruments and observations are changing.",
-            metric_code=MetricCode.historical,
-            rows=table_data,
-            warnings=self._format_warnings(warnings),
-        )
-
 
 class _PolcalQualityMixin:
     """Mixin Class supporting the building of polcal-specific metrics."""

{dkist_processing_common-13.0.4 → dkist_processing_common-13.0.5}/dkist_processing_common/tests/test_assemble_quality.py

@@ -76,46 +76,6 @@ def quality_metrics_old(dataframe_json) -> list[Metric]:
     Quality metric data - this OLD FORMAT is being phased out
     """
     metrics = [
-        Metric(
-            {
-                "x_values": ["2021-01-01T01:01:01", "2021-01-01T02:01:01"],
-                "y_values": [5, 6],
-                "series_name": "",
-            },
-            ["QUALITY_NOISE"],
-        ),
-        Metric(
-            {
-                "x_values": ["2021-01-01T01:01:01", "2021-01-01T02:01:01"],
-                "y_values": [1, 2],
-                "series_name": "I",
-            },
-            ["QUALITY_SENSITIVITY", "STOKES_I"],
-        ),
-        Metric(
-            {
-                "x_values": ["2021-01-01T01:01:01", "2021-01-01T02:01:01"],
-                "y_values": [3, 4],
-                "series_name": "Q",
-            },
-            ["QUALITY_SENSITIVITY", "STOKES_Q"],
-        ),
-        Metric(
-            {
-                "x_values": ["2021-01-01T01:01:01", "2021-01-01T02:01:01"],
-                "y_values": [5, 6],
-                "series_name": "U",
-            },
-            ["QUALITY_SENSITIVITY", "STOKES_U"],
-        ),
-        Metric(
-            {
-                "x_values": ["2021-01-01T01:01:01", "2021-01-01T02:01:01"],
-                "y_values": [7, 8],
-                "series_name": "V",
-            },
-            ["QUALITY_SENSITIVITY", "STOKES_V"],
-        ),
         Metric(
             {"task_type": "dark", "total_frames": 100, "frames_not_used": 7}, ["QUALITY_TASK_TYPES"]
         ),
@@ -170,15 +130,6 @@ def quality_metrics_old(dataframe_json) -> list[Metric]:
             },
             ["QUALITY_POLCAL_EFFICIENCY", "QUALITY_TASK_BEAM 1"],
         ),
-        Metric({"name": "metric 1", "warnings": ["warning 1"]}, ["QUALITY_RANGE"]),
-        Metric({"name": "metric 2", "warnings": ["warning 2"]}, ["QUALITY_RANGE"]),
-        Metric({"name": "metric 3", "warnings": ["warning 3"]}, ["QUALITY_RANGE"]),
-        Metric({"name": "hist 1", "value": 7, "warnings": None}, ["QUALITY_HISTORICAL"]),
-        Metric({"name": "hist 2", "value": "abc", "warnings": None}, ["QUALITY_HISTORICAL"]),
-        Metric(
-            {"name": "hist 3", "value": 9.35, "warnings": "warning for historical metric 3"},
-            ["QUALITY_HISTORICAL"],
-        ),
         Metric(
             {
                 "input_wavelength_nm": [1001.0, 1002.0, 1003.0, 1004.0],
@@ -378,8 +329,6 @@ def plot_data_expected() -> Callable[[str], bool]:
         "Root Mean Square (RMS) Across Frame - GAIN",
         "Fried Parameter",
         "Light Level",
-        "Noise Estimation",
-        "Sensitivity",
     }
 
     def expected(name: str) -> bool:
@@ -414,7 +363,6 @@ def table_data_expected() -> Callable[[str], bool]:
         "Data Source Health",
         "Frame Counts",
         "PolCal Global Calibration Unit Fit - Beam 1",
-        "Historical Comparisons",
     }
 
     def expected(name: str) -> bool:
@@ -517,8 +465,6 @@ def warnings_expected() -> Callable[[str], bool]:
         "Frame Counts",
         "PolCal Global Calibration Unit Fit - Beam 1",
         "PolCal Modulation Efficiency - Beam 1",
-        "Range checks",
-        "Historical Comparisons",
     }
 
     def expected(name: str) -> bool:
@@ -619,8 +565,8 @@ def test_assemble_quality_data(
     quality_data = list(
         chain.from_iterable(task.read(tags=Tag.quality_data(), decoder=json_decoder))
     )
-    # With polcal, this would be 20, but the polcal metrics are not included with this task
-    assert len(quality_data) == 16
+    # With polcal, this would be 16, but the polcal metrics are not included with this task
+    assert len(quality_data) == 12
     for metric_data in quality_data:
         rm: ReportMetric = ReportMetric.from_dict(metric_data)
         assert isinstance(rm.name, str)
@@ -671,8 +617,8 @@ def test_assemble_quality_data_for_polcal(
     quality_data = list(
         chain.from_iterable(task.read(tags=Tag.quality_data(), decoder=json_decoder))
     )
-    # this is 20 with polcal
-    assert len(quality_data) == 20
+    # this is 16 with polcal
+    assert len(quality_data) == 16
     for metric_data in quality_data:
         rm: ReportMetric = ReportMetric.from_dict(metric_data)
         assert isinstance(rm.name, str)