dkist-processing-common 11.7.0rc6__tar.gz → 11.8.0rc1__tar.gz

{dkist_processing_common-11.7.0rc6 → dkist_processing_common-11.8.0rc1}/CHANGELOG.rst

@@ -1,3 +1,28 @@
+v11.7.0 (2025-10-09)
+====================
+
+Features
+--------
+
+- Add new buds to parsing for what will become the dataset extras. (`#267 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/267>`__)
+- Add new bud type TaskContributingIdsBud, based on ContributingIdsBud, for for specific task types. (`#267 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/267>`__)
+- Add new bud types TaskAverageBud and TaskBeginDateBud, which is based on new TaskDatetimeBudBase. (`#267 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/267>`__)
+
+
+Removals
+--------
+
+- Remove IdBud, which is just a TaskUniqueBud with the task set to observe, and therefore is not needed. (`#267 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/267>`__)
+- Remove the `_set_metadata_key_value` method from FitsAccessBase.  Instead of using `setattr`, attributes
+  are assigned explicitly in access classes using `MetadataKey` members in place of header key strings. (`#267 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/267>`__)
+
+
+Misc
+----
+
+- Rename TimeFlowerBase and TaskTimeBudBase to RoundTimeFlowerBase and TaskRoundTimeBudBase, respectively. (`#267 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/267>`__)
+
+
 v11.6.0 (2025-09-26)
 ====================
 

{dkist_processing_common-11.7.0rc6 → dkist_processing_common-11.8.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 11.7.0rc6
+Version: 11.8.0rc1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -57,9 +57,9 @@ Requires-Dist: pytest; extra == "docs"
 Requires-Dist: towncrier<22.12.0; extra == "docs"
 Requires-Dist: dkist-sphinx-theme; extra == "docs"
 Provides-Extra: inventory
-Requires-Dist: dkist-inventory<2.0,>=1.6.0; extra == "inventory"
+Requires-Dist: dkist-inventory<2.0,>=1.11.0; extra == "inventory"
 Provides-Extra: asdf
-Requires-Dist: dkist-inventory[asdf]<2.0,>=1.6.0; extra == "asdf"
+Requires-Dist: dkist-inventory[asdf]<2.0,>=1.11.0; extra == "asdf"
 Provides-Extra: quality
 Requires-Dist: dkist-quality<3.0,>=2.0.0; extra == "quality"
 

dkist_processing_common-11.8.0rc1/changelog/245.feature.1.rst

@@ -0,0 +1 @@
+Add parameters and their associated values relevant to a particular processing pipeline run to the metadata ASDF file generated in trial workflows.

{dkist_processing_common-11.7.0rc6 → dkist_processing_common-11.8.0rc1}/dkist_processing_common/tasks/trial_catalog.py

@@ -5,15 +5,18 @@ import logging
 from datetime import datetime
 from itertools import chain
 from pathlib import Path
+from typing import Any
 from typing import Generator
 from uuid import uuid4
 
 from dkist_processing_common.codecs.asdf import asdf_fileobj_encoder
+from dkist_processing_common.codecs.basemodel import basemodel_decoder
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.json import json_encoder
 from dkist_processing_common.codecs.path import path_decoder
 from dkist_processing_common.codecs.quality import quality_data_decoder
 from dkist_processing_common.models.fits_access import FitsAccessBase
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.output_data_base import OutputDataBase
 
@@ -138,11 +141,15 @@ class CreateTrialAsdf(OutputDataBase):
 
     def run(self) -> None:
         """Generate an ASDF file simulating the ASDF file that would be produced when cataloging the dataset."""
+        with self.telemetry_span("Collate input dataset parameters"):
+            parameters = self.parse_input_dataset_parameters()
+
         with self.telemetry_span("Generate ASDF tree"):
             tree = asdf_tree_from_filenames(
                 filenames=self.absolute_output_frame_paths,
                 hdu=1,  # compressed
                 relative_to=self.scratch.workflow_base_path,
+                parameters=parameters,
             )
 
         trial_history = [
@@ -169,6 +176,37 @@ class CreateTrialAsdf(OutputDataBase):
                     ),
                 )
 
+    def parse_input_dataset_parameters(self) -> list[dict[str, Any]]:
+        """
+        Return the parameters associated with the dataset.
+
+        Returns
+        -------
+        list[dict[str, Any]]
+            A list of dictionaries, each containing a parameter name and its values.
+
+        Raises
+        ------
+        ValueError
+            If there is not exactly one ``InputDatasetPartDocumentList`` found.
+        """
+        part_docs_iter = self.read(
+            tags=Tag.input_dataset_parameters(),
+            decoder=basemodel_decoder,
+            model=InputDatasetPartDocumentList,
+        )
+        docs = list(part_docs_iter)
+
+        if not docs:
+            logger.warning("No parameter list decoded from files")
+            return []
+
+        if len(docs) > 1:
+            raise ValueError(f"Expected 1 parameter list, found {len(docs)}")
+
+        parameters = docs[0].model_dump(by_alias=True).get("doc_list", [])
+        return parameters
+
 
 class CreateTrialQualityReport(OutputDataBase):
     """

{dkist_processing_common-11.7.0rc6 → dkist_processing_common-11.8.0rc1}/dkist_processing_common/tests/mock_metadata_store.py

@@ -6,6 +6,7 @@ import json
 from abc import ABC
 from abc import abstractmethod
 from datetime import datetime
+from datetime import timedelta
 from pathlib import Path
 from uuid import uuid4
 
@@ -234,3 +235,41 @@ def fake_gql_client():
     Convenience fixture for default mock GQL client. To customize, use fake_gql_client_factory.
     """
     return fake_gql_client_factory()
+
+
+def input_dataset_parameters_part_factory(
+    parameter_count: int = 1,
+    parameter_value_count: int = 1,
+    has_date: bool = False,
+    has_file: bool = False,
+) -> list[dict]:
+    """Create a mock InputDatasetPartDocumentList with parameters."""
+    result = [
+        {
+            "parameterName": uuid4().hex[:6],
+            "parameterValues": [
+                {"parameterValueId": i, "parameterValue": json.dumps(uuid4().hex)}
+                for i in range(parameter_value_count)
+            ],
+        }
+        for _ in range(parameter_count)
+    ]
+    if has_date:
+        base = datetime(2018, 9, 14, 0, 0, 0)  # This date is before any possible start dates
+        for parameter_index, data in enumerate(result):
+            for item in data["parameterValues"]:
+                dt = base + timedelta(days=parameter_index)
+                item["parameterValueStartDate"] = dt.isoformat()
+    if has_file:
+        for data in result:
+            param_list = data["parameterValues"]
+            for item in param_list:
+                item["parameterValue"] = json.dumps(
+                    {
+                        "__file__": {
+                            "bucket": "data",
+                            "objectKey": f"parameters/{data['parameterName']}/{uuid4().hex}.dat",
+                        }
+                    }
+                )
+    return result

{dkist_processing_common-11.7.0rc6 → dkist_processing_common-11.8.0rc1}/dkist_processing_common/tests/test_input_dataset.py

@@ -1,5 +1,4 @@
 import json
-from datetime import datetime
 from typing import Any
 from uuid import uuid4
 
@@ -8,6 +7,7 @@ import pytest
 from dkist_processing_common.codecs.basemodel import basemodel_decoder
 from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.tests.mock_metadata_store import input_dataset_parameters_part_factory
 
 
 def input_dataset_frames_part_factory(bucket_count: int = 1) -> list[dict]:
@@ -25,42 +25,6 @@ def flatten_frame_parts(frame_parts: list[dict]) -> list[tuple[str, str]]:
     return result
 
 
-def input_dataset_parameters_part_factory(
-    parameter_count: int = 1,
-    parameter_value_count: int = 1,
-    has_date: bool = False,
-    has_file: bool = False,
-) -> list[dict]:
-    result = [
-        {
-            "parameterName": uuid4().hex[:6],
-            "parameterValues": [
-                {"parameterValueId": i, "parameterValue": json.dumps(uuid4().hex)}
-                for i in range(parameter_value_count)
-            ],
-        }
-        for _ in range(parameter_count)
-    ]
-    if has_date:
-        for data in result:
-            param_list = data["parameterValues"]
-            for item in param_list:
-                item["parameterValueStartDate"] = datetime(2022, 9, 14).isoformat()
-    if has_file:
-        for data in result:
-            param_list = data["parameterValues"]
-            for item in param_list:
-                item["parameterValue"] = json.dumps(
-                    {
-                        "__file__": {
-                            "bucket": "data",
-                            "objectKey": f"parameters/{data['parameterName']}/{uuid4().hex}.dat",
-                        }
-                    }
-                )
-    return result
-
-
 @pytest.mark.parametrize(
     "input_dataset_parts",
     [

{dkist_processing_common-11.7.0rc6 → dkist_processing_common-11.8.0rc1}/dkist_processing_common/tests/test_trial_catalog.py

@@ -12,14 +12,30 @@ from sqids import Sqids
 
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.asdf import asdf_decoder
+from dkist_processing_common.codecs.basemodel import basemodel_encoder
 from dkist_processing_common.codecs.bytes import bytes_decoder
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
 from dkist_processing_common.codecs.json import json_decoder
 from dkist_processing_common.codecs.quality import quality_data_encoder
+from dkist_processing_common.models.input_dataset import InputDatasetParameter
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks import CreateTrialAsdf
 from dkist_processing_common.tasks import CreateTrialDatasetInventory
 from dkist_processing_common.tasks import CreateTrialQualityReport
+from dkist_processing_common.tests.mock_metadata_store import input_dataset_parameters_part_factory
+
+
+@pytest.fixture()
+def mock_input_dataset_parts() -> InputDatasetPartDocumentList:
+    """An InputDatasetPartDocumentList with two parameters, each with one value and a date."""
+    raw = input_dataset_parameters_part_factory(
+        parameter_count=2,
+        parameter_value_count=1,
+        has_date=True,
+        has_file=False,
+    )
+    return InputDatasetPartDocumentList.model_validate({"doc_list": raw})
 
 
 @pytest.fixture()
@@ -41,6 +57,24 @@ def scratch_with_l1_frames(recipe_run_id, tmp_path) -> WorkflowFileSystem:
         scratch.write(
             file_obj, tags=[Tag.output(), Tag.frame()], relative_path=f"{uuid4().hex}.dat"
         )
+
+    return scratch
+
+
+@pytest.fixture()
+def scratch_with_l1_frames_and_parameters(
+    scratch_with_l1_frames, mock_input_dataset_parts
+) -> WorkflowFileSystem:
+    """Scratch instance for a recipe run id with tagged L1 frames and input parameters."""
+    scratch = scratch_with_l1_frames
+
+    # Write validated Pydantic model bytes expected by InputDatasetPartDocumentList
+    file_obj = basemodel_encoder(mock_input_dataset_parts)
+    scratch.write(
+        file_obj,
+        tags=Tag.input_dataset_parameters(),
+        relative_path=f"{uuid4().hex}.json",
+    )
     return scratch
 
 
@@ -85,6 +119,22 @@ def create_trial_asdf_task(
     task._purge()
 
 
+@pytest.fixture(scope="function")
+def create_trial_asdf_task_with_params(
+    recipe_run_id, tmp_path, scratch_with_l1_frames_and_parameters, fake_constants_db
+) -> CreateTrialAsdf:
+    """An instance of CreateTrialAsdf with L1 frames and input parameters tagged in scratch."""
+    task = CreateTrialAsdf(
+        recipe_run_id=recipe_run_id,
+        workflow_name="trial_asdf",
+        workflow_version="trial_asdf_version",
+    )
+    task.scratch = scratch_with_l1_frames_and_parameters
+    task.constants._update(fake_constants_db)
+    yield task
+    task._purge()
+
+
 @pytest.fixture()
 def create_trial_quality_report_task(
     recipe_run_id, tmp_path, fake_constants_db
@@ -143,25 +193,32 @@ def test_create_trial_dataset_inventory(create_trial_dataset_inventory_task):
     assert len(inventory) > 20  # a bunch
 
 
-def test_create_trial_asdf(create_trial_asdf_task, recipe_run_id):
+@pytest.mark.parametrize("with_params", [False, True], ids=["no_params", "with_params"])
+def test_create_trial_asdf(with_params, request, recipe_run_id, mock_input_dataset_parts):
     """
     :Given: An instance of CreateTrialAsdf with L1 frames tagged in scratch
     :When: CreateTrialAsdf is run
     :Then: An asdf file for the dataset is tagged in scratch
     """
-    task = create_trial_asdf_task
+    task = request.getfixturevalue(
+        "create_trial_asdf_task_with_params" if with_params else "create_trial_asdf_task"
+    )
     # When
     task()
+
     # Then
     asdf_tags = [Tag.output(), Tag.asdf()]
     filepaths = list(task.scratch.find_all(tags=asdf_tags))
     assert len(filepaths) == 1
     dataset_id = Sqids(min_length=6, alphabet=ascii_uppercase).encode([recipe_run_id])
     assert filepaths[0].name == f"INSTRUMENT_L1_20240416T160000_{dataset_id}_metadata.asdf"
+
     results = list(task.read(tags=asdf_tags, decoder=asdf_decoder))
     assert len(results) == 1
+
     tree = results[0]
     assert isinstance(tree, dict)
+
     for file_name in tree["dataset"].files.filenames:
         # This is a slightly better than check that `not Path(file_name).is_absolute()` because it confirms
         # we've correctly stripped the path of *all* parents (not just those that start at root).
@@ -169,6 +226,19 @@ def test_create_trial_asdf(create_trial_asdf_task, recipe_run_id):
         # `scratch.workflow_base_path`
         assert Path(file_name).name == file_name
 
+    # Only check parameters when present
+    ds = tree["dataset"]
+    assert "parameters" in ds.meta
+    parameters = ds.meta["parameters"]
+    assert isinstance(parameters, list)
+    if with_params:
+        assert parameters, f"ASDF tree must include input parameters: {parameters}"
+        assert len(parameters) == len(mock_input_dataset_parts.doc_list)
+        for param in parameters:
+            assert InputDatasetParameter.model_validate(param) in mock_input_dataset_parts.doc_list
+    else:
+        assert ds.meta["parameters"] == []
+
 
 def test_create_trial_quality_report(create_trial_quality_report_task):
     """

{dkist_processing_common-11.7.0rc6 → dkist_processing_common-11.8.0rc1}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 11.7.0rc6
+Version: 11.8.0rc1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -57,9 +57,9 @@ Requires-Dist: pytest; extra == "docs"
 Requires-Dist: towncrier<22.12.0; extra == "docs"
 Requires-Dist: dkist-sphinx-theme; extra == "docs"
 Provides-Extra: inventory
-Requires-Dist: dkist-inventory<2.0,>=1.6.0; extra == "inventory"
+Requires-Dist: dkist-inventory<2.0,>=1.11.0; extra == "inventory"
 Provides-Extra: asdf
-Requires-Dist: dkist-inventory[asdf]<2.0,>=1.6.0; extra == "asdf"
+Requires-Dist: dkist-inventory[asdf]<2.0,>=1.11.0; extra == "asdf"
 Provides-Extra: quality
 Requires-Dist: dkist-quality<3.0,>=2.0.0; extra == "quality"
 

{dkist_processing_common-11.7.0rc6 → dkist_processing_common-11.8.0rc1}/dkist_processing_common.egg-info/SOURCES.txt

@@ -7,12 +7,7 @@ README.rst
 bitbucket-pipelines.yml
 pyproject.toml
 changelog/.gitempty
-changelog/267.feature.1.rst
-changelog/267.feature.2.rst
-changelog/267.feature.rst
-changelog/267.misc.rst
-changelog/267.removal.1.rst
-changelog/267.removal.rst
+changelog/245.feature.1.rst
 dkist_processing_common/__init__.py
 dkist_processing_common/config.py
 dkist_processing_common/manual.py

{dkist_processing_common-11.7.0rc6 → dkist_processing_common-11.8.0rc1}/dkist_processing_common.egg-info/requires.txt

@@ -24,7 +24,7 @@ sunpy>=3.0.0
 talus==1.3.4
 
 [asdf]
-dkist-inventory[asdf]<2.0,>=1.6.0
+dkist-inventory[asdf]<2.0,>=1.11.0
 
 [docs]
 sphinx
@@ -36,7 +36,7 @@ towncrier<22.12.0
 dkist-sphinx-theme
 
 [inventory]
-dkist-inventory<2.0,>=1.6.0
+dkist-inventory<2.0,>=1.11.0
 
 [quality]
 dkist-quality<3.0,>=2.0.0

{dkist_processing_common-11.7.0rc6 → dkist_processing_common-11.8.0rc1}/pyproject.toml

@@ -76,10 +76,10 @@ docs = [
   "dkist-sphinx-theme",
 ]
 inventory = [
-  "dkist-inventory >= 1.6.0, <2.0",
+  "dkist-inventory >= 1.11.0, <2.0",
 ]
 asdf = [
-  "dkist-inventory[asdf] >= 1.6.0, <2.0",
+  "dkist-inventory[asdf] >= 1.11.0, <2.0",
 ]
 quality = [
   "dkist-quality >= 2.0.0, <3.0",

dkist_processing_common-11.7.0rc6/changelog/267.feature.1.rst

@@ -1 +0,0 @@
-Add new bud types TaskAverageBud and TaskBeginDateBud, which is based on new TaskDatetimeBudBase.

dkist_processing_common-11.7.0rc6/changelog/267.feature.2.rst

@@ -1 +0,0 @@
-Add new bud type TaskContributingIdsBud, based on ContributingIdsBud, for for specific task types.

dkist_processing_common-11.7.0rc6/changelog/267.feature.rst

@@ -1 +0,0 @@
-Add new buds to parsing for what will become the dataset extras.

dkist_processing_common-11.7.0rc6/changelog/267.misc.rst

@@ -1 +0,0 @@
-Rename TimeFlowerBase and TaskTimeBudBase to RoundTimeFlowerBase and TaskRoundTimeBudBase, respectively.

dkist_processing_common-11.7.0rc6/changelog/267.removal.1.rst

@@ -1,2 +0,0 @@
-Remove the `_set_metadata_key_value` method from FitsAccessBase.  Instead of using `setattr`, attributes
-are assigned explicitly in access classes using `MetadataKey` members in place of header key strings.

dkist_processing_common-11.7.0rc6/changelog/267.removal.rst

@@ -1 +0,0 @@
-Remove IdBud, which is just a TaskUniqueBud with the task set to observe, and therefore is not needed.