dkist-processing-common 11.1.0rc1__tar.gz → 11.2.1__tar.gz

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/CHANGELOG.rst

@@ -1,3 +1,58 @@
+v11.2.1 (2025-08-12)
+====================
+
+Bugfixes
+--------
+
+- Add `multi_plot_data` to quality creation mutation when submitting quality report data. (`#262 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/262>`__)
+
+
+Misc
+----
+
+- Explicitly set color and ordering of lines in wavecal metric plots.
+  This gets around issues with order not being preserved in the quality database. (`#262 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/262>`__)
+- Update database name for vertical multi-pane plot metric data. It's now called "multi_plot_data". (`#262 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/262>`__)
+
+
+v11.2.0 (2025-07-18)
+====================
+
+Bugfixes
+--------
+
+- Move the writing of spectral line keys in L1 data to later in the process, after the instruments have had a chance to rewrite the WCS info. (`#252 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/252>`__)
+
+
+v11.1.0 (2025-07-15)
+====================
+
+Misc
+----
+
+- The graphql and input dataset models are now set to validate whenever attributes are assigned, including after instantiation. (`#255 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/255>`__)
+- Create a factory for custom `FakeGQLClient` used in tests.  The factory and associated default returns live in a
+  new `mock_metadata_store.py` module in the tests directory.  `FakeGQLClient` is now a test fixture and does not need to
+  be imported in tests. (`#255 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/255>`__)
+- Fix a Pydantic warning in the graphql model.  Because we validate the `RecipeRunConfiguration` as a JSON dictionary and
+  convert it to a model after validation, we need a serializer so Pydantic knows what to expect back out. (`#258 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/258>`__)
+
+
+v11.0.1 (2025-07-10)
+====================
+
+Bugfixes
+--------
+
+- Ignore NaN and Inf values when storing the quality results of a wavelength calibration fit. (`#260 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/260>`__)
+
+
+Misc
+----
+
+- Removing usages of `pkg_resources`, which is on track to be deprecated. (`#254 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/254>`__)
+
+
 v11.0.0 (2025-07-02)
 ====================
 

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 11.1.0rc1
+Version: 11.2.1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -61,7 +61,7 @@ Requires-Dist: dkist-inventory<2.0,>=1.6.0; extra == "inventory"
 Provides-Extra: asdf
 Requires-Dist: dkist-inventory[asdf]<2.0,>=1.6.0; extra == "asdf"
 Provides-Extra: quality
-Requires-Dist: dkist-quality<2.0,>=1.3.0; extra == "quality"
+Requires-Dist: dkist-quality<3.0,>=2.0.0; extra == "quality"
 
 dkist-processing-common
 =======================

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/dkist_processing_common/models/graphql.py

@@ -1,6 +1,7 @@
 """GraphQL Data models for the metadata store api."""
 from pydantic import BaseModel
 from pydantic import ConfigDict
+from pydantic import field_serializer
 from pydantic import field_validator
 from pydantic import Json
 
@@ -129,6 +130,10 @@ class RecipeRunResponse(GraphqlBaseModel):
             return RecipeRunConfiguration()
         return RecipeRunConfiguration.model_validate(value)
 
+    @field_serializer("configuration")
+    def _serialize_as_basemodel(self, config: RecipeRunConfiguration):
+        return config.model_dump()
+
 
 class RecipeRunMutationResponse(GraphqlBaseModel):
     """Recipe run mutation response."""
@@ -185,6 +190,7 @@ class QualityCreation(GraphqlBaseModel):
     warnings: str | None = None
     # JSON objects
     plotData: str | None = None
+    multiPlotData: str | None = None
     tableData: str | None = None
     histogramData: str | None = None
     modmatData: str | None = None

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/dkist_processing_common/models/quality.py

@@ -112,7 +112,7 @@ class ReportMetric(BaseModel):
     facet: str | None = None
     statement: str | list[str] | None = None
     plot_data: Plot2D | list[Plot2D] | None = None
-    vertical_multi_pane_plot_data: VerticalMultiPanePlot2D | None = None
+    multi_plot_data: VerticalMultiPanePlot2D | None = None
     histogram_data: PlotHistogram | list[PlotHistogram] | None = None
     table_data: SimpleTable | list[SimpleTable] | None = None
     modmat_data: ModulationMatrixHistograms | None = None

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/dkist_processing_common/tasks/assemble_movie.py

@@ -2,10 +2,10 @@
 import logging
 from abc import ABC
 from abc import abstractmethod
+from importlib.resources import files
 from typing import Literal
 
 import numpy as np
-import pkg_resources
 from matplotlib import colormaps
 from moviepy import VideoClip
 from PIL import Image
@@ -66,7 +66,7 @@ class AssembleMovie(WorkflowTaskBase, ABC):
     MINIMUM_DURATION = 10  # seconds
     MAXIMUM_DURATION = 60  # seconds
     FPS = 15
-    FONT_FILE = pkg_resources.resource_filename("dkist_processing_common", "fonts/Lato-Regular.ttf")
+    FONT_FILE = files("dkist_processing_common").joinpath("fonts/Lato-Regular.ttf")
     TEXT_MARGIN_PX = 5
     MPL_COLOR_MAP = "viridis"
 

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/dkist_processing_common/tasks/mixin/metadata_store.py

@@ -171,6 +171,7 @@ class MetadataStoreMixin:
                 warnings=json.dumps(metric.get("warnings")),
                 # JSON objects
                 plotData=json.dumps(metric.get("plot_data"), cls=QualityDataEncoder),
+                multiPlotData=json.dumps(metric.get("multi_plot_data"), cls=QualityDataEncoder),
                 tableData=json.dumps(metric.get("table_data"), cls=QualityDataEncoder),
                 histogramData=json.dumps(metric.get("histogram_data"), cls=QualityDataEncoder),
                 modmatData=json.dumps(metric.get("modmat_data"), cls=QualityDataEncoder),

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/dkist_processing_common/tasks/mixin/quality/_metrics.py

@@ -1365,13 +1365,22 @@ class _WavecalQualityMixin:
         wcs = WCS(best_fit_header)
         best_fit_wavelength = wcs.spectral.pixel_to_world(np.arange(input_spectrum.size))
 
+        finite_idx = (
+            np.isfinite(input_wavelength)
+            * np.isfinite(input_spectrum)
+            * np.isfinite(best_fit_wavelength)
+            * np.isfinite(best_fit_atlas)
+            * np.isfinite(normalized_residuals)
+            * np.isfinite(weight_data)
+        )
+
         data = {
-            "input_wavelength_nm": input_wavelength.to_value(u.nm).tolist(),
-            "input_spectrum": input_spectrum.tolist(),
-            "best_fit_wavelength_nm": best_fit_wavelength.to_value(u.nm).tolist(),
-            "best_fit_atlas": best_fit_atlas.tolist(),
-            "normalized_residuals": normalized_residuals.tolist(),
-            "weights": None if weights is None else weight_data.tolist(),
+            "input_wavelength_nm": input_wavelength.to_value(u.nm)[finite_idx].tolist(),
+            "input_spectrum": input_spectrum[finite_idx].tolist(),
+            "best_fit_wavelength_nm": best_fit_wavelength.to_value(u.nm)[finite_idx].tolist(),
+            "best_fit_atlas": best_fit_atlas[finite_idx].tolist(),
+            "normalized_residuals": normalized_residuals[finite_idx].tolist(),
+            "weights": None if weights is None else weight_data[finite_idx].tolist(),
         }
 
         self._record_values(values=data, tags=[Tag.quality(MetricCode.wavecal_fit)])
@@ -1392,10 +1401,20 @@ class _WavecalQualityMixin:
             "Input Spectrum": [input_wave_list, input_spectrum_list],
             "Best Fit Atlas": [best_fit_wave_list, best_fit_atlas_list],
         }
+
+        # Set the colors and zorder here manually because the JSON-ization of the quality data means we can't be sure of
+        # the order these will be plotted in and thus can't rely on the default color-cycler in `dkist-quality`.
         fit_plot_kwargs = {
-            "Best Fit Observations": {"ls": "-", "lw": 4, "alpha": 0.8, "ms": 0},
-            "Input Spectrum": {"ls": "-", "alpha": 0.4, "ms": 0},
-            "Best Fit Atlas": {"color": "k", "ls": "-", "ms": 0},
+            "Best Fit Observations": {
+                "ls": "-",
+                "lw": 4,
+                "alpha": 0.8,
+                "ms": 0,
+                "color": "#FAA61C",
+                "zorder": 2,
+            },
+            "Input Spectrum": {"ls": "-", "alpha": 0.4, "ms": 0, "color": "#1E317A", "zorder": 2.1},
+            "Best Fit Atlas": {"color": "k", "ls": "-", "ms": 0, "zorder": 2.2},
         }
 
         fit_plot = Plot2D(
@@ -1449,7 +1468,7 @@ class _WavecalQualityMixin:
             "The top plot shows the input and best-fit spectra along with the best-fit atlas, which is "
             "a combination of Solar and Telluric spectra. The bottom plot shows the fit residuals.",
             metric_code=MetricCode.wavecal_fit,
-            vertical_multi_pane_plot_data=full_plot,
+            multi_plot_data=full_plot,
         )
 
         return metric.model_dump()

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/dkist_processing_common/tasks/write_l1.py

@@ -230,7 +230,13 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         header = self.add_contributing_id_headers(header=header)
         header["MANPROCD"] = self.workflow_had_manual_intervention
         header["PRODUCT"] = self.compute_product_id(header["IDSOBSID"], header["PROCTYPE"])
-        # Spectral line keywords
+        return header
+
+    def add_spectral_line_headers(
+        self,
+        header: fits.Header,
+    ) -> fits.Header:
+        """Add datacenter table keys relating to spectral lines."""
         wavelength_range = self.get_wavelength_range(header=header)
         spectral_lines = get_spectral_lines(
             wavelength_min=wavelength_range.min,
@@ -364,6 +370,8 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         header = self.add_doc_headers(header=header)
         # Add the dataset headers (abstract - implement in instrument task)
         header = self.add_dataset_headers(header=header, stokes=stokes_param)
+        # Add the spectral line headers
+        header = self.add_spectral_line_headers(header=header)
         # Remove any headers not contained in spec 214
         header = remove_spec_122_keys_and_spec_214_l0_keys(input_headers=header)
         # Remove any keys referring to axes that don't exist

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/dkist_processing_common/tests/test_assemble_quality.py

@@ -259,11 +259,11 @@ def plot_data_expected() -> Callable[[str], bool]:
 
 
 @pytest.fixture()
-def vertical_multi_pane_plot_data_expected() -> Callable[[str], bool]:
+def multi_plot_data_expected() -> Callable[[str], bool]:
     """
     Tightly coupled with quality_metrics fixture and resultant report metric name
     """
-    # names where vertical_multi_pane_plot_data is expected to be populated
+    # names where multi_plot_data is expected to be populated
     names = {"Wavelength Calibration Results"}
 
     def expected(name: str) -> bool:
@@ -451,7 +451,7 @@ def test_assemble_quality_data(
     assemble_quality_data_task,
     recipe_run_id,
     plot_data_expected,
-    vertical_multi_pane_plot_data_expected,
+    multi_plot_data_expected,
     table_data_expected,
     modmat_data_expected,
     histogram_data_expected,
@@ -481,8 +481,8 @@ def test_assemble_quality_data(
         assert isinstance(rm.description, str)
         if plot_data_expected(rm.name):
             assert rm.plot_data
-        if vertical_multi_pane_plot_data_expected(rm.name):
-            assert rm.vertical_multi_pane_plot_data
+        if multi_plot_data_expected(rm.name):
+            assert rm.multi_plot_data
         if table_data_expected(rm.name):
             assert rm.table_data
         if modmat_data_expected(rm.name):

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/dkist_processing_common/tests/test_quality_mixin.py

@@ -114,7 +114,7 @@ def test_create_2d_plot_with_datetime_metric(quality_task):
         "facet",
         "statement",
         "plot_data",
-        "vertical_multi_pane_plot_data",
+        "multi_plot_data",
         "histogram_data",
         "table_data",
         "modmat_data",
@@ -1201,12 +1201,16 @@ def wavecal_input_wavelength() -> u.Quantity:
 
 @pytest.fixture(scope="session")
 def wavecal_input_spectrum(wavecal_input_wavelength) -> np.ndarray:
-    return (wavecal_input_wavelength.value - wavecal_input_wavelength.size // 2) ** 2 + 10.0
+    spec = (wavecal_input_wavelength.value - wavecal_input_wavelength.size // 2) ** 2 + 10.0
+    spec[spec.size // 2] = np.nan
+    return spec
 
 
 @pytest.fixture(scope="session")
 def wavecal_weights(wavecal_input_wavelength) -> np.ndarray:
-    return np.arange(wavecal_input_wavelength.size)
+    weights = np.arange(wavecal_input_wavelength.size, dtype=float)
+    weights[0] = np.inf
+    return weights
 
 
 @pytest.fixture(scope="session")
@@ -1313,7 +1317,7 @@ def test_build_wavecal_results(quality_task, wavecal_data_json):
     assert metric["raincloud_data"] is None
     assert metric["warnings"] is None
 
-    multi_plot_data = metric["vertical_multi_pane_plot_data"]
+    multi_plot_data = metric["multi_plot_data"]
     assert multi_plot_data["match_x_axes"] is True
     assert multi_plot_data["no_gap"] is True
     assert (
@@ -1335,9 +1339,16 @@ def test_build_wavecal_results(quality_task, wavecal_data_json):
         "Best Fit Atlas": [[1001.5, 1002.6, 1003.7, 1004.8], [1.0, 1.0, 0.4, 1.0]],
     }
     assert fit_plot["plot_kwargs"] == {
-        "Input Spectrum": {"ls": "-", "alpha": 0.4, "ms": 0},
-        "Best Fit Observations": {"ls": "-", "lw": 4, "alpha": 0.8, "ms": 0},
-        "Best Fit Atlas": {"color": "k", "ls": "-", "ms": 0},
+        "Input Spectrum": {"ls": "-", "alpha": 0.4, "ms": 0, "color": "#1E317A", "zorder": 2.1},
+        "Best Fit Observations": {
+            "ls": "-",
+            "lw": 4,
+            "alpha": 0.8,
+            "ms": 0,
+            "color": "#FAA61C",
+            "zorder": 2.0,
+        },
+        "Best Fit Atlas": {"color": "k", "ls": "-", "ms": 0, "zorder": 2.2},
     }
 
     residuals_plot = plot_list[1]

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/dkist_processing_common/tests/test_workflow_task_base.py

@@ -2,12 +2,11 @@ import json
 import logging
 import re
 import tomllib
-from dataclasses import asdict
+from importlib.metadata import version
 from pathlib import Path
 from string import ascii_uppercase
 
 import pytest
-from pkg_resources import get_distribution
 from sqids import Sqids
 
 import dkist_processing_common
@@ -272,7 +271,7 @@ def test_library_versions(provenance_task, package_dependencies):
     #   installed packages.
     for package in package_dependencies:
         assert package in libraries
-        assert libraries[package] == get_distribution(package).version
+        assert libraries[package] == version(package)
 
 
 def test_record_provenance(provenance_task):

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/dkist_processing_common/tests/test_write_l1.py

@@ -677,7 +677,7 @@ def test_spectral_line_keys(write_l1_task, mocker, fake_gql_client):
         new=fake_gql_client,
     )
     task, _, header = write_l1_task
-    header = task.add_datacenter_headers(header=header, hdu_size=1024, stokes="I")
+    header = task.add_spectral_line_headers(header=header)
     assert header["SPECLN01"] == "Fe XIII (1079.8 nm)"
     assert header["SPECLN02"] == "He I (1083.0 nm)"
     assert header["NSPECLNS"] == 2

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 11.1.0rc1
+Version: 11.2.1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -61,7 +61,7 @@ Requires-Dist: dkist-inventory<2.0,>=1.6.0; extra == "inventory"
 Provides-Extra: asdf
 Requires-Dist: dkist-inventory[asdf]<2.0,>=1.6.0; extra == "asdf"
 Provides-Extra: quality
-Requires-Dist: dkist-quality<2.0,>=1.3.0; extra == "quality"
+Requires-Dist: dkist-quality<3.0,>=2.0.0; extra == "quality"
 
 dkist-processing-common
 =======================

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/dkist_processing_common.egg-info/SOURCES.txt

@@ -7,7 +7,6 @@ README.rst
 bitbucket-pipelines.yml
 pyproject.toml
 changelog/.gitempty
-changelog/255.misc.rst
 dkist_processing_common/__init__.py
 dkist_processing_common/config.py
 dkist_processing_common/manual.py

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/dkist_processing_common.egg-info/requires.txt

@@ -39,7 +39,7 @@ dkist-sphinx-theme
 dkist-inventory<2.0,>=1.6.0
 
 [quality]
-dkist-quality<2.0,>=1.3.0
+dkist-quality<3.0,>=2.0.0
 
 [test]
 pytest

{dkist_processing_common-11.1.0rc1 → dkist_processing_common-11.2.1}/pyproject.toml

@@ -82,7 +82,7 @@ asdf = [
   "dkist-inventory[asdf] >= 1.6.0, <2.0",
 ]
 quality = [
-  "dkist-quality >= 1.3.0, <2.0",
+  "dkist-quality >= 2.0.0, <3.0",
 ]
 
 [tool.pytest.ini_options]

dkist_processing_common-11.1.0rc1/changelog/255.misc.rst

@@ -1,3 +0,0 @@
-Create a factory for custom `FakeGQLClient` used in tests.  The factory and associated default returns live in a
-new `mock_metadata_store.py` module in the tests directory.  `FakeGQLClient` is now a test fixture and does not need to
-be imported in tests.