dkist-processing-common 11.7.0rc6__py3-none-any.whl → 11.8.0rc1__py3-none-any.whl

changelog/245.feature.1.rst

@@ -0,0 +1 @@
+Add parameters and their associated values relevant to a particular processing pipeline run to the metadata ASDF file generated in trial workflows.

dkist_processing_common/tasks/trial_catalog.py

@@ -5,15 +5,18 @@ import logging
 from datetime import datetime
 from itertools import chain
 from pathlib import Path
+from typing import Any
 from typing import Generator
 from uuid import uuid4
 
 from dkist_processing_common.codecs.asdf import asdf_fileobj_encoder
+from dkist_processing_common.codecs.basemodel import basemodel_decoder
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.json import json_encoder
 from dkist_processing_common.codecs.path import path_decoder
 from dkist_processing_common.codecs.quality import quality_data_decoder
 from dkist_processing_common.models.fits_access import FitsAccessBase
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.output_data_base import OutputDataBase
 
@@ -138,11 +141,15 @@ class CreateTrialAsdf(OutputDataBase):
 
     def run(self) -> None:
         """Generate an ASDF file simulating the ASDF file that would be produced when cataloging the dataset."""
+        with self.telemetry_span("Collate input dataset parameters"):
+            parameters = self.parse_input_dataset_parameters()
+
         with self.telemetry_span("Generate ASDF tree"):
             tree = asdf_tree_from_filenames(
                 filenames=self.absolute_output_frame_paths,
                 hdu=1,  # compressed
                 relative_to=self.scratch.workflow_base_path,
+                parameters=parameters,
             )
 
         trial_history = [
@@ -169,6 +176,37 @@ class CreateTrialAsdf(OutputDataBase):
                     ),
                 )
 
+    def parse_input_dataset_parameters(self) -> list[dict[str, Any]]:
+        """
+        Return the parameters associated with the dataset.
+
+        Returns
+        -------
+        list[dict[str, Any]]
+            A list of dictionaries, each containing a parameter name and its values.
+
+        Raises
+        ------
+        ValueError
+            If there is not exactly one ``InputDatasetPartDocumentList`` found.
+        """
+        part_docs_iter = self.read(
+            tags=Tag.input_dataset_parameters(),
+            decoder=basemodel_decoder,
+            model=InputDatasetPartDocumentList,
+        )
+        docs = list(part_docs_iter)
+
+        if not docs:
+            logger.warning("No parameter list decoded from files")
+            return []
+
+        if len(docs) > 1:
+            raise ValueError(f"Expected 1 parameter list, found {len(docs)}")
+
+        parameters = docs[0].model_dump(by_alias=True).get("doc_list", [])
+        return parameters
+
 
 class CreateTrialQualityReport(OutputDataBase):
     """

dkist_processing_common/tests/mock_metadata_store.py

@@ -6,6 +6,7 @@ import json
 from abc import ABC
 from abc import abstractmethod
 from datetime import datetime
+from datetime import timedelta
 from pathlib import Path
 from uuid import uuid4
 
@@ -234,3 +235,41 @@ def fake_gql_client():
     Convenience fixture for default mock GQL client. To customize, use fake_gql_client_factory.
     """
     return fake_gql_client_factory()
+
+
+def input_dataset_parameters_part_factory(
+    parameter_count: int = 1,
+    parameter_value_count: int = 1,
+    has_date: bool = False,
+    has_file: bool = False,
+) -> list[dict]:
+    """Create a mock InputDatasetPartDocumentList with parameters."""
+    result = [
+        {
+            "parameterName": uuid4().hex[:6],
+            "parameterValues": [
+                {"parameterValueId": i, "parameterValue": json.dumps(uuid4().hex)}
+                for i in range(parameter_value_count)
+            ],
+        }
+        for _ in range(parameter_count)
+    ]
+    if has_date:
+        base = datetime(2018, 9, 14, 0, 0, 0)  # This date is before any possible start dates
+        for parameter_index, data in enumerate(result):
+            for item in data["parameterValues"]:
+                dt = base + timedelta(days=parameter_index)
+                item["parameterValueStartDate"] = dt.isoformat()
+    if has_file:
+        for data in result:
+            param_list = data["parameterValues"]
+            for item in param_list:
+                item["parameterValue"] = json.dumps(
+                    {
+                        "__file__": {
+                            "bucket": "data",
+                            "objectKey": f"parameters/{data['parameterName']}/{uuid4().hex}.dat",
+                        }
+                    }
+                )
+    return result

dkist_processing_common/tests/test_input_dataset.py

@@ -1,5 +1,4 @@
 import json
-from datetime import datetime
 from typing import Any
 from uuid import uuid4
 
@@ -8,6 +7,7 @@ import pytest
 from dkist_processing_common.codecs.basemodel import basemodel_decoder
 from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.tests.mock_metadata_store import input_dataset_parameters_part_factory
 
 
 def input_dataset_frames_part_factory(bucket_count: int = 1) -> list[dict]:
@@ -25,42 +25,6 @@ def flatten_frame_parts(frame_parts: list[dict]) -> list[tuple[str, str]]:
     return result
 
 
-def input_dataset_parameters_part_factory(
-    parameter_count: int = 1,
-    parameter_value_count: int = 1,
-    has_date: bool = False,
-    has_file: bool = False,
-) -> list[dict]:
-    result = [
-        {
-            "parameterName": uuid4().hex[:6],
-            "parameterValues": [
-                {"parameterValueId": i, "parameterValue": json.dumps(uuid4().hex)}
-                for i in range(parameter_value_count)
-            ],
-        }
-        for _ in range(parameter_count)
-    ]
-    if has_date:
-        for data in result:
-            param_list = data["parameterValues"]
-            for item in param_list:
-                item["parameterValueStartDate"] = datetime(2022, 9, 14).isoformat()
-    if has_file:
-        for data in result:
-            param_list = data["parameterValues"]
-            for item in param_list:
-                item["parameterValue"] = json.dumps(
-                    {
-                        "__file__": {
-                            "bucket": "data",
-                            "objectKey": f"parameters/{data['parameterName']}/{uuid4().hex}.dat",
-                        }
-                    }
-                )
-    return result
-
-
 @pytest.mark.parametrize(
     "input_dataset_parts",
     [

dkist_processing_common/tests/test_trial_catalog.py

@@ -12,14 +12,30 @@ from sqids import Sqids
 
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.asdf import asdf_decoder
+from dkist_processing_common.codecs.basemodel import basemodel_encoder
 from dkist_processing_common.codecs.bytes import bytes_decoder
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
 from dkist_processing_common.codecs.json import json_decoder
 from dkist_processing_common.codecs.quality import quality_data_encoder
+from dkist_processing_common.models.input_dataset import InputDatasetParameter
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks import CreateTrialAsdf
 from dkist_processing_common.tasks import CreateTrialDatasetInventory
 from dkist_processing_common.tasks import CreateTrialQualityReport
+from dkist_processing_common.tests.mock_metadata_store import input_dataset_parameters_part_factory
+
+
+@pytest.fixture()
+def mock_input_dataset_parts() -> InputDatasetPartDocumentList:
+    """An InputDatasetPartDocumentList with two parameters, each with one value and a date."""
+    raw = input_dataset_parameters_part_factory(
+        parameter_count=2,
+        parameter_value_count=1,
+        has_date=True,
+        has_file=False,
+    )
+    return InputDatasetPartDocumentList.model_validate({"doc_list": raw})
 
 
 @pytest.fixture()
@@ -41,6 +57,24 @@ def scratch_with_l1_frames(recipe_run_id, tmp_path) -> WorkflowFileSystem:
         scratch.write(
             file_obj, tags=[Tag.output(), Tag.frame()], relative_path=f"{uuid4().hex}.dat"
         )
+
+    return scratch
+
+
+@pytest.fixture()
+def scratch_with_l1_frames_and_parameters(
+    scratch_with_l1_frames, mock_input_dataset_parts
+) -> WorkflowFileSystem:
+    """Scratch instance for a recipe run id with tagged L1 frames and input parameters."""
+    scratch = scratch_with_l1_frames
+
+    # Write validated Pydantic model bytes expected by InputDatasetPartDocumentList
+    file_obj = basemodel_encoder(mock_input_dataset_parts)
+    scratch.write(
+        file_obj,
+        tags=Tag.input_dataset_parameters(),
+        relative_path=f"{uuid4().hex}.json",
+    )
     return scratch
 
 
@@ -85,6 +119,22 @@ def create_trial_asdf_task(
     task._purge()
 
 
+@pytest.fixture(scope="function")
+def create_trial_asdf_task_with_params(
+    recipe_run_id, tmp_path, scratch_with_l1_frames_and_parameters, fake_constants_db
+) -> CreateTrialAsdf:
+    """An instance of CreateTrialAsdf with L1 frames and input parameters tagged in scratch."""
+    task = CreateTrialAsdf(
+        recipe_run_id=recipe_run_id,
+        workflow_name="trial_asdf",
+        workflow_version="trial_asdf_version",
+    )
+    task.scratch = scratch_with_l1_frames_and_parameters
+    task.constants._update(fake_constants_db)
+    yield task
+    task._purge()
+
+
 @pytest.fixture()
 def create_trial_quality_report_task(
     recipe_run_id, tmp_path, fake_constants_db
@@ -143,25 +193,32 @@ def test_create_trial_dataset_inventory(create_trial_dataset_inventory_task):
     assert len(inventory) > 20  # a bunch
 
 
-def test_create_trial_asdf(create_trial_asdf_task, recipe_run_id):
+@pytest.mark.parametrize("with_params", [False, True], ids=["no_params", "with_params"])
+def test_create_trial_asdf(with_params, request, recipe_run_id, mock_input_dataset_parts):
     """
     :Given: An instance of CreateTrialAsdf with L1 frames tagged in scratch
     :When: CreateTrialAsdf is run
     :Then: An asdf file for the dataset is tagged in scratch
     """
-    task = create_trial_asdf_task
+    task = request.getfixturevalue(
+        "create_trial_asdf_task_with_params" if with_params else "create_trial_asdf_task"
+    )
     # When
     task()
+
     # Then
     asdf_tags = [Tag.output(), Tag.asdf()]
     filepaths = list(task.scratch.find_all(tags=asdf_tags))
     assert len(filepaths) == 1
     dataset_id = Sqids(min_length=6, alphabet=ascii_uppercase).encode([recipe_run_id])
     assert filepaths[0].name == f"INSTRUMENT_L1_20240416T160000_{dataset_id}_metadata.asdf"
+
     results = list(task.read(tags=asdf_tags, decoder=asdf_decoder))
     assert len(results) == 1
+
     tree = results[0]
     assert isinstance(tree, dict)
+
     for file_name in tree["dataset"].files.filenames:
         # This is a slightly better than check that `not Path(file_name).is_absolute()` because it confirms
         # we've correctly stripped the path of *all* parents (not just those that start at root).
@@ -169,6 +226,19 @@ def test_create_trial_asdf(create_trial_asdf_task, recipe_run_id):
         # `scratch.workflow_base_path`
         assert Path(file_name).name == file_name
 
+    # Only check parameters when present
+    ds = tree["dataset"]
+    assert "parameters" in ds.meta
+    parameters = ds.meta["parameters"]
+    assert isinstance(parameters, list)
+    if with_params:
+        assert parameters, f"ASDF tree must include input parameters: {parameters}"
+        assert len(parameters) == len(mock_input_dataset_parts.doc_list)
+        for param in parameters:
+            assert InputDatasetParameter.model_validate(param) in mock_input_dataset_parts.doc_list
+    else:
+        assert ds.meta["parameters"] == []
+
 
 def test_create_trial_quality_report(create_trial_quality_report_task):
     """

{dkist_processing_common-11.7.0rc6.dist-info → dkist_processing_common-11.8.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 11.7.0rc6
+Version: 11.8.0rc1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -57,9 +57,9 @@ Requires-Dist: pytest; extra == "docs"
 Requires-Dist: towncrier<22.12.0; extra == "docs"
 Requires-Dist: dkist-sphinx-theme; extra == "docs"
 Provides-Extra: inventory
-Requires-Dist: dkist-inventory<2.0,>=1.6.0; extra == "inventory"
+Requires-Dist: dkist-inventory<2.0,>=1.11.0; extra == "inventory"
 Provides-Extra: asdf
-Requires-Dist: dkist-inventory[asdf]<2.0,>=1.6.0; extra == "asdf"
+Requires-Dist: dkist-inventory[asdf]<2.0,>=1.11.0; extra == "asdf"
 Provides-Extra: quality
 Requires-Dist: dkist-quality<3.0,>=2.0.0; extra == "quality"
 

{dkist_processing_common-11.7.0rc6.dist-info → dkist_processing_common-11.8.0rc1.dist-info}/RECORD

@@ -1,10 +1,5 @@
 changelog/.gitempty,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-changelog/267.feature.1.rst,sha256=G5Xl7EwiRKAtftSUvalfF7iUL-WgqIvnQ3k6wyTYzaw,98
-changelog/267.feature.2.rst,sha256=1z0TqX3159m-wFvEgElBjUp0XPWQvmqMlVlf4BzO-J8,99
-changelog/267.feature.rst,sha256=g4bBK3EFQiQljW9lQrFh6rVOL8C6yxBYbXI4WOH8NE0,65
-changelog/267.misc.rst,sha256=ciUqUesBbdxh41euyLhyD3Bl6IVT2TSIGeOg3LFjIvg,105
-changelog/267.removal.1.rst,sha256=qiGl1FAy9ioLwfCh8EvM788wjFbOY3jM3eUJHPOzpu8,207
-changelog/267.removal.rst,sha256=ecpctwJ-AE59sfbsTy-j3GYaPaN8h6oWphXUMNQ42Yw,103
+changelog/245.feature.1.rst,sha256=BBwCgALdFPqX7ZI0Uj-z9piaGplVKbgErZ6LuMH3qbA,148
 dkist_processing_common/__init__.py,sha256=GQ9EBnYhkOnt-qODclAoLS_g5YVhurxfg1tjVtI9rDI,320
 dkist_processing_common/config.py,sha256=kkFm-Q6jGGoV3Zdf3PSfk3xNYd2vSH0J2R8dBBav3Og,5218
 dkist_processing_common/manual.py,sha256=bIVVyLsbXMh-g_2L3kGROL-1TtJe0_XviHsp7Br31x8,7023
@@ -70,7 +65,7 @@ dkist_processing_common/tasks/parse_l0_input_data.py,sha256=KguXT0Xavynu7C8NFMjs
 dkist_processing_common/tasks/quality_metrics.py,sha256=cvGF6tJ8yAvxOvkeG3tWxYwL885BrFW5X3V7_MSzL-A,12481
 dkist_processing_common/tasks/teardown.py,sha256=rwT9lWINVDF11-az_nx-Z5ykMTX_SJCchobpU6sErgk,2360
 dkist_processing_common/tasks/transfer_input_data.py,sha256=DAYfS-B1o-iBT9MXU-TiJG4Hv05Z0c_JzPrnFgvnK9g,5786
-dkist_processing_common/tasks/trial_catalog.py,sha256=iAaMT_oLnupA1O3xAtqVjsqRY5f_hyvMps-fXg6KlHU,8729
+dkist_processing_common/tasks/trial_catalog.py,sha256=43XN4Hz4uNBGEps3O5RLfFHRfyo9FZPzMwkcapniztQ,10049
 dkist_processing_common/tasks/trial_output_data.py,sha256=CPMXXODvN5RTcu9bTF8v6AXciCl212EWP6qTiARvUNk,6837
 dkist_processing_common/tasks/write_l1.py,sha256=Xy834RTp3F95kLcW4ba5gfHMUocfZd82ZQQKnvQcP2M,23204
 dkist_processing_common/tasks/mixin/__init__.py,sha256=-g-DQbU7m1bclJYuFe3Yh757V-35GIDTbstardKQ7nU,68
@@ -83,7 +78,7 @@ dkist_processing_common/tasks/mixin/quality/_base.py,sha256=qt9TZZ140skFWFmabrjl
 dkist_processing_common/tasks/mixin/quality/_metrics.py,sha256=k0hlthVbW7Jso9q_P0-hWGfp190kQO_oUQyGBRvxgqo,60626
 dkist_processing_common/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dkist_processing_common/tests/conftest.py,sha256=Tm-Yq956EAafpDtu1d7JjdVY0Unp9e4z9ak-yf4wnH4,22320
-dkist_processing_common/tests/mock_metadata_store.py,sha256=fbCvSk1-s0ojN6l538RWodPW7dx6k4eXqipemnHKO0Y,8248
+dkist_processing_common/tests/mock_metadata_store.py,sha256=-uzwbLeRn19_Pq1hr97R4hpUD5K5sv8Zy1U8gG0GdSQ,9657
 dkist_processing_common/tests/test_assemble_movie.py,sha256=dyVhowxB-Kc6GuxlDs74UrPtK9fwdUL7y5haA3Bidz0,4065
 dkist_processing_common/tests/test_assemble_quality.py,sha256=-F22jMY6mPy65VZ1TZY2r1vsxMXOPmZHArGx70OD3BA,17832
 dkist_processing_common/tests/test_base.py,sha256=gsyBG2R6Ufx7CzbHeGMagUwM9yCfpN4gCSZ6-aH2q48,6643
@@ -94,7 +89,7 @@ dkist_processing_common/tests/test_dkist_location.py,sha256=-_OoSw4SZDLFyIuOltHv
 dkist_processing_common/tests/test_fits_access.py,sha256=a50B4IAAH5NH5zeudTqyy0b5uWKJwJuzQLUdK1LoOHM,12832
 dkist_processing_common/tests/test_flower_pot.py,sha256=X9_UI3maa3ZQncV3jYHgovWnawDsdEkEB5vw6EAB96o,3151
 dkist_processing_common/tests/test_fried_parameter.py,sha256=iXtlQIifZ6cDOkEi-YDgP3oAlss2loq08Uohgvy1byQ,1295
-dkist_processing_common/tests/test_input_dataset.py,sha256=pQ01rWAkQ2XQojyHWzAqeOdrMXshNcgEVL5I_9bBTdo,9610
+dkist_processing_common/tests/test_input_dataset.py,sha256=wnQbZxBYywG5CEXces2WWk6I0QA7HjStaYSTVVbe5r0,8499
 dkist_processing_common/tests/test_interservice_bus.py,sha256=QrBeZ8dh497h6nxA8-aVUIGDcSj8y9DIXIk9I_HkXr0,3001
 dkist_processing_common/tests/test_interservice_bus_mixin.py,sha256=IptJkW7Qeu2Y742NKXEgkok2VdS600keLgCD3Y9iw3A,4131
 dkist_processing_common/tests/test_manual_processing.py,sha256=iHF7yQPlar9niYAGXtFv28Gw3Undlds38yMfszk4ccY,1037
@@ -113,7 +108,7 @@ dkist_processing_common/tests/test_task_parsing.py,sha256=2_OOmeZQWD17XAd_ECYmod
 dkist_processing_common/tests/test_teardown.py,sha256=DaliHSGsiQBZaFkf5wb3XBo6rHNPmx2bmQtVymYeBN4,5601
 dkist_processing_common/tests/test_transfer_input_data.py,sha256=eyAAWXpTHQ8aew87-MncWpYBn4DAZrTSOL3LvlQfR5Q,12611
 dkist_processing_common/tests/test_transfer_l1_output_data.py,sha256=PVGDJBEUk4kAeu8ivrhlCE7yd29R18t9kZLFx-mpBwY,2063
-dkist_processing_common/tests/test_trial_catalog.py,sha256=Efq4yP1_0jccCFTi0_inyf4utC-oJVpGoAZ13fBP7I0,6778
+dkist_processing_common/tests/test_trial_catalog.py,sha256=CxjtVABE5Fw2EvyXR56IJ3PPi9QvEOjccH0OzzRWk30,9424
 dkist_processing_common/tests/test_trial_output_data.py,sha256=YwmSfhNsSG-kdnvNlq7Ku0PNi_onTvU6uxn_nLiAKZw,12008
 dkist_processing_common/tests/test_workflow_task_base.py,sha256=LTVusltNrsGUOvw9G323am4CXebgE4tJhP6gZCcS0CQ,10457
 dkist_processing_common/tests/test_write_l1.py,sha256=alN-lozKEm6vKNdhtvzjnuPqv-NjHyUg16Op7SkMH-c,27964
@@ -125,7 +120,7 @@ docs/landing_page.rst,sha256=aPAuXFhBx73lEZ59B6E6JXxkK0LlxzD0n-HXqHrfumQ,746
 docs/make.bat,sha256=mBAhtURwhQ7yc95pqwJzlhqBSvRknr1aqZ5s8NKvdKs,4513
 docs/requirements.txt,sha256=Kbl_X4c7RQZw035YTeNB63We6I7pvXFU4T0Uflp2yDY,29
 licenses/LICENSE.rst,sha256=piZaQplkzOMmH1NXg6QIdo9wwo9pPCoHkvm2-DmH76E,1462
-dkist_processing_common-11.7.0rc6.dist-info/METADATA,sha256=6JeOGbgzvzWIZH1VFNxUDvdoxhCDulcTuAjq8aEIebQ,13323
-dkist_processing_common-11.7.0rc6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dkist_processing_common-11.7.0rc6.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
-dkist_processing_common-11.7.0rc6.dist-info/RECORD,,
+dkist_processing_common-11.8.0rc1.dist-info/METADATA,sha256=k1cSXjEW_NNR3JwR7h55s9SFLH-H2k4XfT-RigHlAsQ,13325
+dkist_processing_common-11.8.0rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dkist_processing_common-11.8.0rc1.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
+dkist_processing_common-11.8.0rc1.dist-info/RECORD,,

changelog/267.feature.1.rst

@@ -1 +0,0 @@
-Add new bud types TaskAverageBud and TaskBeginDateBud, which is based on new TaskDatetimeBudBase.

changelog/267.feature.2.rst

@@ -1 +0,0 @@
-Add new bud type TaskContributingIdsBud, based on ContributingIdsBud, for for specific task types.

changelog/267.feature.rst

@@ -1 +0,0 @@
-Add new buds to parsing for what will become the dataset extras.

changelog/267.misc.rst

@@ -1 +0,0 @@
-Rename TimeFlowerBase and TaskTimeBudBase to RoundTimeFlowerBase and TaskRoundTimeBudBase, respectively.

changelog/267.removal.1.rst

@@ -1,2 +0,0 @@
-Remove the `_set_metadata_key_value` method from FitsAccessBase.  Instead of using `setattr`, attributes
-are assigned explicitly in access classes using `MetadataKey` members in place of header key strings.

changelog/267.removal.rst

@@ -1 +0,0 @@
-Remove IdBud, which is just a TaskUniqueBud with the task set to observe, and therefore is not needed.