dkist-processing-common 10.6.2__tar.gz → 10.6.4__tar.gz

{dkist_processing_common-10.6.2 → dkist_processing_common-10.6.4}/CHANGELOG.rst

@@ -1,3 +1,21 @@
+v10.6.4 (2025-03-31)
+====================
+
+Bugfixes
+--------
+
+- Allow input dataset IDs to be conditionally written into the L1 data headers. (`#243 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/243>`__)
+
+
+v10.6.3 (2025-03-27)
+====================
+
+Bugfixes
+--------
+
+- Fix a bug where transfer input dataset fails if any of the input dataset part documents are missing. (`#241 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/241>`__)
+- Pydantic model validator now returns the validated instance. (`#240 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/240>`__)
+
 v10.6.2 (2025-03-19)
 ====================
 

{dkist_processing_common-10.6.2 → dkist_processing_common-10.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 10.6.2
+Version: 10.6.4
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-10.6.2 → dkist_processing_common-10.6.4}/dkist_processing_common/models/wavelength.py

@@ -26,3 +26,4 @@ class WavelengthRange(BaseModel):
         """Validate that the max wavelength is greater than the min wavelength."""
         if self.min > self.max:
             raise ValueError("min is greater than max.  Values may be reversed.")
+        return self

{dkist_processing_common-10.6.2 → dkist_processing_common-10.6.4}/dkist_processing_common/tasks/transfer_input_data.py

@@ -20,12 +20,17 @@ class TransferL0Data(WorkflowTaskBase, GlobusMixin, InputDatasetMixin):
 
     def download_input_dataset(self):
         """Get the input dataset document parts and save it to scratch with the appropriate tags."""
-        if doc := self.metadata_store_input_dataset_observe_frames.inputDatasetPartDocument:
-            self.write(doc, tags=Tag.input_dataset_observe_frames(), encoder=json_encoder)
-        if doc := self.metadata_store_input_dataset_calibration_frames.inputDatasetPartDocument:
-            self.write(doc, tags=Tag.input_dataset_calibration_frames(), encoder=json_encoder)
-        if doc := self.metadata_store_input_dataset_parameters.inputDatasetPartDocument:
-            self.write(doc, tags=Tag.input_dataset_parameters(), encoder=json_encoder)
+        if observe_frames := self.metadata_store_input_dataset_observe_frames:
+            observe_doc = observe_frames.inputDatasetPartDocument
+            self.write(observe_doc, tags=Tag.input_dataset_observe_frames(), encoder=json_encoder)
+        if calibration_frames := self.metadata_store_input_dataset_calibration_frames:
+            calibration_doc = calibration_frames.inputDatasetPartDocument
+            self.write(
+                calibration_doc, tags=Tag.input_dataset_calibration_frames(), encoder=json_encoder
+            )
+        if parameters := self.metadata_store_input_dataset_parameters:
+            parameters_doc = parameters.inputDatasetPartDocument
+            self.write(parameters_doc, tags=Tag.input_dataset_parameters(), encoder=json_encoder)
 
     def format_transfer_items(
         self, input_dataset_objects: list[InputDatasetObject]

{dkist_processing_common-10.6.2 → dkist_processing_common-10.6.4}/dkist_processing_common/tasks/write_l1.py

@@ -214,11 +214,12 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         header["CADMAX"] = self.constants.maximum_cadence
         header["CADVAR"] = self.constants.variance_cadence
         # Keywords to support reprocessing
-        if ids_par_id := self.metadata_store_input_dataset_parameters.inputDatasetPartId:
-            header["IDSPARID"] = ids_par_id
-        header["IDSOBSID"] = self.metadata_store_input_dataset_observe_frames.inputDatasetPartId
-        if ids_cal_id := self.metadata_store_input_dataset_calibration_frames.inputDatasetPartId:
-            header["IDSCALID"] = ids_cal_id
+        if parameters := self.metadata_store_input_dataset_parameters:
+            header["IDSPARID"] = parameters.inputDatasetPartId
+        if observe_frames := self.metadata_store_input_dataset_observe_frames:
+            header["IDSOBSID"] = observe_frames.inputDatasetPartId
+        if calibration_frames := self.metadata_store_input_dataset_calibration_frames:
+            header["IDSCALID"] = calibration_frames.inputDatasetPartId
         header["WKFLNAME"] = self.workflow_name
         header["WKFLVERS"] = self.workflow_version
         header = self.add_contributing_id_headers(header=header)

{dkist_processing_common-10.6.2 → dkist_processing_common-10.6.4}/dkist_processing_common/tests/test_transfer_input_data.py

@@ -6,6 +6,7 @@ import pytest
 
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.json import json_decoder
+from dkist_processing_common.models.graphql import InputDatasetRecipeRunResponse
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.transfer_input_data import TransferL0Data
 from dkist_processing_common.tests.conftest import create_parameter_files
@@ -17,10 +18,21 @@ class TransferL0DataTask(TransferL0Data):
         ...
 
 
-@pytest.fixture
-def transfer_l0_data_task(recipe_run_id, tmp_path, mocker):
+class FakeGQLClientMissingInputDatasetPart(FakeGQLClient):
+    """Same metadata mocker with calibration input dataset part missing."""
+
+    def execute_gql_query(self, **kwargs):
+        original_response = super().execute_gql_query(**kwargs)
+        # Remove calibration frames part if getting InputDatasetRecipeRunResponse:
+        if kwargs.get("query_response_cls") == InputDatasetRecipeRunResponse:
+            del original_response[0].recipeInstance.inputDataset.inputDatasetInputDatasetParts[2]
+        return original_response
+
+
+def _transfer_l0_data_task_with_client(recipe_run_id, tmp_path, mocker, client_cls):
     mocker.patch(
-        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
+        new=client_cls,
     )
     with TransferL0DataTask(
         recipe_run_id=recipe_run_id,
@@ -35,6 +47,18 @@ def transfer_l0_data_task(recipe_run_id, tmp_path, mocker):
         task._purge()
 
 
+@pytest.fixture
+def transfer_l0_data_task(recipe_run_id, tmp_path, mocker):
+    yield from _transfer_l0_data_task_with_client(recipe_run_id, tmp_path, mocker, FakeGQLClient)
+
+
+@pytest.fixture
+def transfer_l0_data_task_missing_part(recipe_run_id, tmp_path, mocker):
+    yield from _transfer_l0_data_task_with_client(
+        recipe_run_id, tmp_path, mocker, FakeGQLClientMissingInputDatasetPart
+    )
+
+
 def test_download_dataset(transfer_l0_data_task):
     """
     :Given: a TransferL0Data task with a valid input dataset
@@ -63,6 +87,29 @@ def test_download_dataset(transfer_l0_data_task):
     assert parameters_doc_from_file == expected_parameters_doc
 
 
+def test_download_dataset_missing_part(transfer_l0_data_task_missing_part):
+    """
+    :Given: a TransferL0Data task with a valid input dataset without calibration frames
+    :When: downloading the dataset documents from the metadata store
+    :Then: the correct number of documents are written to disk
+    """
+    # Given
+    task = transfer_l0_data_task_missing_part
+    # When
+    task.download_input_dataset()
+    # Then
+    observe_doc_from_file = next(
+        task.read(tags=Tag.input_dataset_observe_frames(), decoder=json_decoder)
+    )
+    parameters_doc_from_file = next(
+        task.read(tags=Tag.input_dataset_parameters(), decoder=json_decoder)
+    )
+    with pytest.raises(StopIteration):
+        calibration_doc_from_file = next(
+            task.read(tags=Tag.input_dataset_calibration_frames(), decoder=json_decoder)
+        )
+
+
 def test_format_frame_transfer_items(transfer_l0_data_task):
     """
     :Given: a TransferL0Data task with a downloaded input dataset

{dkist_processing_common-10.6.2 → dkist_processing_common-10.6.4}/dkist_processing_common/tests/test_write_l1.py

@@ -24,6 +24,9 @@ from dkist_processing_common.models.wavelength import WavelengthRange
 from dkist_processing_common.tasks.write_l1 import WriteL1Frame
 from dkist_processing_common.tests.conftest import FakeGQLClient
 from dkist_processing_common.tests.conftest import TILE_SIZE
+from dkist_processing_common.tests.test_transfer_input_data import (
+    FakeGQLClientMissingInputDatasetPart,
+)
 
 
 class FakeGQLClientDefaultRecipeConfiguration(FakeGQLClient):
@@ -484,6 +487,29 @@ def test_reprocessing_keys(write_l1_task, mocker):
         assert header["PRODUCT"] == task.compute_product_id(header["IDSOBSID"], header["PROCTYPE"])
 
 
+def test_missing_input_dataset_part(write_l1_task, mocker):
+    """
+    :Given: a Write_L1 task with a missing calibration frames part
+    :When: running the task
+    :Then: the input dataset part keys are correctly written without throwing an exception
+    """
+    mocker.patch(
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
+        new=FakeGQLClientMissingInputDatasetPart,
+    )
+    task, _, _ = write_l1_task
+    task()
+    files = list(task.read(tags=[Tag.frame(), Tag.output()]))
+    for file in files:
+        header = fits.open(file)[1].header
+        assert header["IDSPARID"] == task.metadata_store_input_dataset_parameters.inputDatasetPartId
+        assert (
+            header["IDSOBSID"]
+            == task.metadata_store_input_dataset_observe_frames.inputDatasetPartId
+        )
+        assert "IDSCALID" not in header
+
+
 @pytest.mark.parametrize(
     "ids_obs_id, proc_type",
     [

{dkist_processing_common-10.6.2 → dkist_processing_common-10.6.4}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 10.6.2
+Version: 10.6.4
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause