dkist-processing-common 14.0.2rc1__tar.gz → 14.0.3__tar.gz

{dkist_processing_common-14.0.2rc1 → dkist_processing_common-14.0.3}/CHANGELOG.rst

@@ -1,3 +1,22 @@
+v14.0.3 (2026-05-05)
+====================
+
+Features
+--------
+
+- Add recipe run configuration information relevant to the metadata ASDF file generated in trial workflows. (`#318 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/318>`__)
+
+
+v14.0.2 (2026-05-04)
+====================
+
+Features
+--------
+
+- Convert TASK_TYPES quality metric to the new methodology. (`#332 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/332>`__)
+- Add `TaskCount` quality metric model. (`#332 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/332>`__)
+
+
 v14.0.1 (2026-04-22)
 ====================
 

{dkist_processing_common-14.0.2rc1 → dkist_processing_common-14.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 14.0.2rc1
+Version: 14.0.3
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -56,9 +56,9 @@ Requires-Dist: sphinx-autoapi!=3.1.0; extra == "docs"
 Requires-Dist: pytest; extra == "docs"
 Requires-Dist: dkist-sphinx-theme; extra == "docs"
 Provides-Extra: inventory
-Requires-Dist: dkist-inventory<2.0,>=1.11.6; extra == "inventory"
+Requires-Dist: dkist-inventory<2.0,>=1.11.7; extra == "inventory"
 Provides-Extra: asdf
-Requires-Dist: dkist-inventory[asdf]<2.0,>=1.11.6; extra == "asdf"
+Requires-Dist: dkist-inventory[asdf]<2.0,>=1.11.7; extra == "asdf"
 Provides-Extra: quality
 Requires-Dist: dkist-quality<3.0,>=2.0.1; extra == "quality"
 

{dkist_processing_common-14.0.2rc1 → dkist_processing_common-14.0.3}/dkist_processing_common/tasks/base.py

@@ -19,13 +19,10 @@ from opentelemetry.metrics import Observation
 
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common._util.tags import TagDB
-from dkist_processing_common.codecs.basemodel import basemodel_encoder
 from dkist_processing_common.codecs.bytes import bytes_encoder
 from dkist_processing_common.codecs.path import path_decoder
 from dkist_processing_common.config import common_configurations
 from dkist_processing_common.models.constants import ConstantsBase
-from dkist_processing_common.models.metric_code import MetricCode
-from dkist_processing_common.models.quality import TaskCount
 from dkist_processing_common.models.tags import StemName
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.models.telemetry import ObservableProgress
@@ -341,25 +338,6 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
         """Remove the association between the given tag(s) and the given path."""
         self.scratch.remove_tags(path, tags)
 
-    def quality_write_intermediate_task_type_counts(
-        self, task_type: str, total_frames: int, frames_not_used: int = 0
-    ):
-        """
-        Write intermediate task type data.
-
-        Parameters
-        ----------
-        task_type: task type as listed in the headers
-        total_frames: total number of frames supplied of the given task type
-        frames_not_used: if some frames aren't used, how many
-        """
-        task_count = TaskCount(
-            task_type=task_type,
-            total_frames=total_frames,
-            frames_not_used=frames_not_used,
-        )
-        self.write(task_count, tags=Tag.quality(MetricCode.task_types), encoder=basemodel_encoder)
-
     def _purge(self):
         """Purge the persistent stores associated with the workflow to which this task belongs e.g. when testing."""
         self.scratch.purge()

{dkist_processing_common-14.0.2rc1 → dkist_processing_common-14.0.3}/dkist_processing_common/tasks/trial_catalog.py

@@ -155,6 +155,12 @@ class CreateTrialAsdf(OutputDataBase):
                 Tag.input_dataset_calibration_frames()
             )
 
+        with self.telemetry_span("Collate recipe run configuration"):
+            recipe_run_config = self.metadata_store_recipe_run.configuration.model_dump(
+                exclude_unset=True,
+                exclude_none=True,
+            )
+
         with self.telemetry_span("Generate ASDF tree"):
             tree = asdf_tree_from_filenames(
                 filenames=self.absolute_output_frame_paths,
@@ -163,6 +169,7 @@ class CreateTrialAsdf(OutputDataBase):
                 parameters=parameters,
                 observation_frames=observation_frames,
                 calibration_frames=calibration_frames,
+                recipe_run_config=recipe_run_config,
             )
 
         trial_history = [

{dkist_processing_common-14.0.2rc1 → dkist_processing_common-14.0.3}/dkist_processing_common/tests/mock_metadata_store.py

@@ -96,7 +96,11 @@ default_parameters_doc = [
     },
 ]
 
-default_recipe_run_configuration = {"tile_size": TILE_SIZE}
+default_recipe_run_configuration = {
+    "tile_size": TILE_SIZE,
+    "destination_bucket": "data",
+    "validate_l1_on_write": True,
+}
 
 
 class Unset:

{dkist_processing_common-14.0.2rc1 → dkist_processing_common-14.0.3}/dkist_processing_common/tests/test_trial_catalog.py

@@ -27,8 +27,12 @@ from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks import CreateTrialAsdf
 from dkist_processing_common.tasks import CreateTrialDatasetInventory
 from dkist_processing_common.tasks import CreateTrialQualityReport
+from dkist_processing_common.tests.mock_metadata_store import RecipeRunResponseMapping
 from dkist_processing_common.tests.mock_metadata_store import default_observe_frames_doc
+from dkist_processing_common.tests.mock_metadata_store import default_recipe_run_configuration
+from dkist_processing_common.tests.mock_metadata_store import fake_gql_client_factory
 from dkist_processing_common.tests.mock_metadata_store import input_dataset_parameters_part_factory
+from dkist_processing_common.tests.mock_metadata_store import make_default_recipe_run_response
 
 
 def _parameter_for_test_comparison(param: dict) -> dict:
@@ -155,20 +159,46 @@ def scratch_with_l1_frames_and_other_frames(
     return scratch
 
 
+@pytest.fixture()
+def patch_default_graphql_client(mocker, fake_gql_client) -> None:
+    """Patch the metadata store client to use the default fake GraphQL client."""
+    mocker.patch(
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
+        new=fake_gql_client,
+    )
+
+
+@pytest.fixture()
+def fake_gql_client_without_recipe_config():
+    """GraphQL client mock that returns a recipe run response without explicit configuration."""
+    recipe_run_response = make_default_recipe_run_response()
+    recipe_run_response.configuration = None
+
+    return fake_gql_client_factory(
+        response_mapping_override=RecipeRunResponseMapping(response=recipe_run_response)
+    )
+
+
+@pytest.fixture()
+def patch_graphql_client_without_recipe_config(
+    mocker, fake_gql_client_without_recipe_config
+) -> None:
+    """Patch the metadata store client to return a recipe run response without config."""
+    mocker.patch(
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
+        new=fake_gql_client_without_recipe_config,
+    )
+
+
 @pytest.fixture()
 def create_trial_dataset_inventory_task(
     recipe_run_id,
     tmp_path,
     scratch_with_l1_frames,
     fake_constants_db,
-    mocker,
-    fake_gql_client,
+    patch_default_graphql_client,
 ) -> CreateTrialDatasetInventory:
     """An instance of CreateTrialDatasetInventory with L1 frames tagged in scratch."""
-    mocker.patch(
-        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
-        new=fake_gql_client,
-    )
     task = CreateTrialDatasetInventory(
         recipe_run_id=recipe_run_id,
         workflow_name="trial_dataset_inventory",
@@ -182,7 +212,7 @@ def create_trial_dataset_inventory_task(
 
 @pytest.fixture()
 def create_trial_asdf_task(
-    recipe_run_id, tmp_path, scratch_with_l1_frames, fake_constants_db
+    recipe_run_id, tmp_path, scratch_with_l1_frames, fake_constants_db, patch_default_graphql_client
 ) -> CreateTrialAsdf:
     """An instance of CreateTrialAsdf with L1 frames tagged in scratch."""
     task = CreateTrialAsdf(
@@ -196,9 +226,33 @@ def create_trial_asdf_task(
     task._purge()
 
 
+@pytest.fixture(scope="function")
+def create_trial_asdf_task_without_recipe_config(
+    recipe_run_id,
+    tmp_path,
+    scratch_with_l1_frames,
+    fake_constants_db,
+    patch_graphql_client_without_recipe_config,
+) -> CreateTrialAsdf:
+    """An instance of CreateTrialAsdf with L1 frames and no recipe run configuration."""
+    task = CreateTrialAsdf(
+        recipe_run_id=recipe_run_id,
+        workflow_name="trial_asdf",
+        workflow_version="trial_asdf_version",
+    )
+    task.scratch = scratch_with_l1_frames
+    task.constants._update(fake_constants_db)
+    yield task
+    task._purge()
+
+
 @pytest.fixture(scope="function")
 def create_trial_asdf_task_with_params(
-    recipe_run_id, tmp_path, scratch_with_l1_frames_and_parameters, fake_constants_db
+    recipe_run_id,
+    tmp_path,
+    scratch_with_l1_frames_and_parameters,
+    fake_constants_db,
+    patch_default_graphql_client,
 ) -> CreateTrialAsdf:
     """An instance of CreateTrialAsdf with L1 frames and input parameters tagged in scratch."""
     task = CreateTrialAsdf(
@@ -214,7 +268,11 @@ def create_trial_asdf_task_with_params(
 
 @pytest.fixture()
 def create_trial_asdf_task_with_frames(
-    recipe_run_id, tmp_path, scratch_with_l1_frames_and_other_frames, fake_constants_db
+    recipe_run_id,
+    tmp_path,
+    scratch_with_l1_frames_and_other_frames,
+    fake_constants_db,
+    patch_default_graphql_client,
 ) -> CreateTrialAsdf:
     """An instance of CreateTrialAsdf with L1 frames and input frame docs tagged in scratch."""
     task = CreateTrialAsdf(
@@ -389,6 +447,38 @@ def test_create_trial_asdf_with_frames(create_trial_asdf_task_with_frames):
     assert set(cal_keys_str) == expected_keys
 
 
+def test_create_trial_asdf_includes_recipe_run_config(create_trial_asdf_task):
+    """
+    :Given: CreateTrialAsdf with recipe run configuration available in metadata store
+    :When: CreateTrialAsdf is run
+    :Then: The ASDF tree contains normalized recipe run configuration metadata
+    """
+    task = create_trial_asdf_task
+    task()
+
+    tree = next(task.read(tags=[Tag.output(), Tag.asdf()], decoder=asdf_decoder))
+    ds = tree["dataset"]
+
+    assert ds.meta["recipe_run_config"] == default_recipe_run_configuration
+
+
+def test_create_trial_asdf_without_recipe_run_config_has_empty_config(
+    create_trial_asdf_task_without_recipe_config,
+):
+    """
+    :Given: CreateTrialAsdf with no recipe run configuration in metadata store
+    :When: CreateTrialAsdf is run
+    :Then: The ASDF tree contains an empty normalized recipe run configuration structure
+    """
+    task = create_trial_asdf_task_without_recipe_config
+    task()
+
+    tree = next(task.read(tags=[Tag.output(), Tag.asdf()], decoder=asdf_decoder))
+    ds = tree["dataset"]
+
+    assert ds.meta["recipe_run_config"] == {}
+
+
 def test_create_trial_quality_report(create_trial_quality_report_task):
     """
     :Given: An instance of CreateTrialQualityReport with tagged quality data

{dkist_processing_common-14.0.2rc1 → dkist_processing_common-14.0.3}/dkist_processing_common/tests/test_workflow_task_base.py

@@ -11,12 +11,10 @@ from sqids import Sqids
 
 import dkist_processing_common
 from dkist_processing_common._util.scratch import WorkflowFileSystem
-from dkist_processing_common.codecs.basemodel import basemodel_decoder
 from dkist_processing_common.codecs.str import str_encoder
 from dkist_processing_common.models.constants import BudName
 from dkist_processing_common.models.constants import ConstantsBase
 from dkist_processing_common.models.graphql import RecipeRunProvenanceMutation
-from dkist_processing_common.models.quality import TaskCount
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.base import WorkflowTaskBase
 
@@ -215,46 +213,6 @@ def test_dataset_id(workflow_data_task):
     assert task.constants.dataset_id == expected_dataset_id
 
 
-@pytest.mark.parametrize(
-    "task_type, total_frames, frames_not_used",
-    [
-        pytest.param("one", 1, 1, id="simple"),
-        pytest.param("two", 2, 0, id="not_used_zero"),
-        pytest.param("three", 3, None, id="not_used_none"),
-    ],
-)
-def test_quality_write_intermediate_task_type_counts(
-    task_type: str, total_frames: int, frames_not_used: int | None, workflow_data_task
-):
-    """
-    Given: Valid intermediate task type count input
-    When: Running WorkflowTaskBase quality_write_intermediate_task_type_counts()
-    Then: A single TaskCount file gets created
-    """
-    # Given
-    task, _, _, _, _ = workflow_data_task
-
-    # When
-    if frames_not_used is not None:
-        task.quality_write_intermediate_task_type_counts(task_type, total_frames, frames_not_used)
-    else:
-        task.quality_write_intermediate_task_type_counts(task_type, total_frames)
-
-    # Then
-    task_counts: list[TaskCount] = list(
-        task.read(tags="QUALITY_TASK_TYPES", decoder=basemodel_decoder, model=TaskCount)
-    )
-    assert isinstance(task_counts, list)
-    assert len(task_counts) == 1
-    task_count = task_counts[0]
-    assert task_count.task_type == str(task_type).upper()
-    assert task_count.total_frames == total_frames
-    if frames_not_used is not None:
-        assert task_count.frames_not_used == frames_not_used
-    else:
-        assert task_count.frames_not_used == 0
-
-
 class ProvenanceTask(WorkflowTaskBase):
     record_provenance = True
 

{dkist_processing_common-14.0.2rc1 → dkist_processing_common-14.0.3}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 14.0.2rc1
+Version: 14.0.3
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -56,9 +56,9 @@ Requires-Dist: sphinx-autoapi!=3.1.0; extra == "docs"
 Requires-Dist: pytest; extra == "docs"
 Requires-Dist: dkist-sphinx-theme; extra == "docs"
 Provides-Extra: inventory
-Requires-Dist: dkist-inventory<2.0,>=1.11.6; extra == "inventory"
+Requires-Dist: dkist-inventory<2.0,>=1.11.7; extra == "inventory"
 Provides-Extra: asdf
-Requires-Dist: dkist-inventory[asdf]<2.0,>=1.11.6; extra == "asdf"
+Requires-Dist: dkist-inventory[asdf]<2.0,>=1.11.7; extra == "asdf"
 Provides-Extra: quality
 Requires-Dist: dkist-quality<3.0,>=2.0.1; extra == "quality"
 

{dkist_processing_common-14.0.2rc1 → dkist_processing_common-14.0.3}/dkist_processing_common.egg-info/SOURCES.txt

@@ -7,7 +7,6 @@ README.rst
 bitbucket-pipelines.yml
 pyproject.toml
 changelog/.gitempty
-changelog/332.feature.rst
 dkist_processing_common/__init__.py
 dkist_processing_common/config.py
 dkist_processing_common/manual.py

{dkist_processing_common-14.0.2rc1 → dkist_processing_common-14.0.3}/dkist_processing_common.egg-info/requires.txt

@@ -24,7 +24,7 @@ sunpy>=3.0.0
 talus==1.3.4
 
 [asdf]
-dkist-inventory[asdf]<2.0,>=1.11.6
+dkist-inventory[asdf]<2.0,>=1.11.7
 
 [docs]
 setuptools
@@ -36,7 +36,7 @@ pytest
 dkist-sphinx-theme
 
 [inventory]
-dkist-inventory<2.0,>=1.11.6
+dkist-inventory<2.0,>=1.11.7
 
 [quality]
 dkist-quality<3.0,>=2.0.1

{dkist_processing_common-14.0.2rc1 → dkist_processing_common-14.0.3}/pyproject.toml

@@ -75,10 +75,10 @@ docs = [
   "dkist-sphinx-theme",
 ]
 inventory = [
-  "dkist-inventory >= 1.11.6, <2.0",
+  "dkist-inventory >= 1.11.7, <2.0",
 ]
 asdf = [
-  "dkist-inventory[asdf] >= 1.11.6, <2.0",
+  "dkist-inventory[asdf] >= 1.11.7, <2.0",
 ]
 quality = [
   "dkist-quality >= 2.0.1, <3.0",

dkist_processing_common-14.0.2rc1/changelog/332.feature.rst

@@ -1 +0,0 @@
-Convert TASK_TYPES quality metric to the new methodology.