dkist-processing-common 10.6.0rc2__py3-none-any.whl → 10.6.1rc1__py3-none-any.whl

changelog/236.misc.1.rst

@@ -0,0 +1 @@
+Change returns from the metadata store queries into Pydantic BaseModel instances.  Remove unnecessary parsing and error checking.

changelog/236.misc.rst

@@ -0,0 +1 @@
+Convert dataclasses in the graphql model to Pydantic BaseModels for additional validation. In the RecipeRunResponse class, configuration is now returned as a dictionary. In the InputDatasetPartResponse class, inputDatasetPartDocument is now returned as a list of dictionaries.

dkist_processing_common/models/graphql.py

@@ -1,24 +1,22 @@
 """GraphQL Data models for the metadata store api."""
-from dataclasses import dataclass
+from pydantic import BaseModel
+from pydantic import Json
 
 
-@dataclass
-class RecipeRunMutation:
+class RecipeRunMutation(BaseModel):
     """Recipe run mutation record."""
 
     recipeRunId: int
     recipeRunStatusId: int
 
 
-@dataclass
-class RecipeRunStatusQuery:
+class RecipeRunStatusQuery(BaseModel):
     """Recipe run status query for the recipeRunStatuses endpoint."""
 
     recipeRunStatusName: str
 
 
-@dataclass
-class RecipeRunStatusMutation:
+class RecipeRunStatusMutation(BaseModel):
     """Recipe run status mutation record."""
 
     recipeRunStatusName: str
@@ -26,38 +24,33 @@ class RecipeRunStatusMutation:
     recipeRunStatusDescription: str
 
 
-@dataclass
-class RecipeRunStatusResponse:
+class RecipeRunStatusResponse(BaseModel):
     """Response to a recipe run status query."""
 
     recipeRunStatusId: int
 
 
-@dataclass
-class InputDatasetPartTypeResponse:
+class InputDatasetPartTypeResponse(BaseModel):
     """Response class for the input dataset part type entity."""
 
     inputDatasetPartTypeName: str
 
 
-@dataclass
-class InputDatasetPartResponse:
+class InputDatasetPartResponse(BaseModel):
     """Response class for the input dataset part entity."""
 
     inputDatasetPartId: int
-    inputDatasetPartDocument: str
+    inputDatasetPartDocument: Json[list[dict]]
     inputDatasetPartType: InputDatasetPartTypeResponse
 
 
-@dataclass
-class InputDatasetInputDatasetPartResponse:
+class InputDatasetInputDatasetPartResponse(BaseModel):
     """Response class for the join entity between input datasets and input dataset parts."""
 
     inputDatasetPart: InputDatasetPartResponse
 
 
-@dataclass
-class InputDatasetResponse:
+class InputDatasetResponse(BaseModel):
     """Input dataset query response."""
 
     inputDatasetId: int
@@ -65,62 +58,54 @@ class InputDatasetResponse:
     inputDatasetInputDatasetParts: list[InputDatasetInputDatasetPartResponse]
 
 
-@dataclass
-class InputDatasetRecipeInstanceResponse:
+class InputDatasetRecipeInstanceResponse(BaseModel):
     """Recipe instance query response."""
 
     inputDataset: InputDatasetResponse
 
 
-@dataclass
-class InputDatasetRecipeRunResponse:
+class InputDatasetRecipeRunResponse(BaseModel):
     """Recipe run query response."""
 
     recipeInstance: InputDatasetRecipeInstanceResponse
 
 
-@dataclass
-class RecipeInstanceResponse:
+class RecipeInstanceResponse(BaseModel):
     """Recipe instance query response."""
 
     recipeId: int
     inputDatasetId: int
 
 
-@dataclass
-class RecipeRunProvenanceResponse:
+class RecipeRunProvenanceResponse(BaseModel):
     """Response for the metadata store recipeRunProvenances and mutations endpoints."""
 
     recipeRunProvenanceId: int
     isTaskManual: bool
 
 
-@dataclass
-class RecipeRunResponse:
+class RecipeRunResponse(BaseModel):
     """Recipe run query response."""
 
     recipeInstance: RecipeInstanceResponse
     recipeInstanceId: int
     recipeRunProvenances: list[RecipeRunProvenanceResponse]
-    configuration: str = None
+    configuration: Json[dict] | None = {}
 
 
-@dataclass
-class RecipeRunMutationResponse:
+class RecipeRunMutationResponse(BaseModel):
     """Recipe run mutation response."""
 
     recipeRunId: int
 
 
-@dataclass
-class RecipeRunQuery:
+class RecipeRunQuery(BaseModel):
     """Query parameters for the metadata store endpoint recipeRuns."""
 
     recipeRunId: int
 
 
-@dataclass
-class DatasetCatalogReceiptAccountMutation:
+class DatasetCatalogReceiptAccountMutation(BaseModel):
     """
     Dataset catalog receipt account mutation record.
 
@@ -132,15 +117,13 @@ class DatasetCatalogReceiptAccountMutation:
     expectedObjectCount: int
 
 
-@dataclass
-class DatasetCatalogReceiptAccountResponse:
+class DatasetCatalogReceiptAccountResponse(BaseModel):
     """Dataset catalog receipt account response for query and mutation endpoints."""
 
     datasetCatalogReceiptAccountId: int
 
 
-@dataclass
-class RecipeRunProvenanceMutation:
+class RecipeRunProvenanceMutation(BaseModel):
     """Recipe run provenance mutation record."""
 
     inputDatasetId: int
@@ -152,8 +135,7 @@ class RecipeRunProvenanceMutation:
     codeVersion: str = None
 
 
-@dataclass
-class QualityCreation:
+class QualityCreation(BaseModel):
     """Quality data creation record."""
 
     datasetId: str
@@ -173,15 +155,13 @@ class QualityCreation:
     efficiencyData: str | None = None
 
 
-@dataclass
-class QualitiesRequest:
+class QualitiesRequest(BaseModel):
     """Query parameters for quality data."""
 
     datasetId: str
 
 
-@dataclass
-class QualityResponse:
+class QualityResponse(BaseModel):
     """Query Response for quality data."""
 
     qualityId: int

dkist_processing_common/tasks/mixin/metadata_store.py

@@ -2,6 +2,7 @@
 import json
 import logging
 from functools import cached_property
+from typing import Literal
 
 from dkist_processing_common._util.graphql import GraphQLClient
 from dkist_processing_common.codecs.quality import QualityDataEncoder
@@ -26,8 +27,6 @@ from dkist_processing_common.models.graphql import RecipeRunStatusResponse
 
 logger = logging.getLogger(__name__)
 
-input_dataset_part_document_type_hint = list | dict | str | int | float | None
-
 
 class MetadataStoreMixin:
     """Mixin for a WorkflowDataTaskBase which implements Metadata Store access functionality."""
@@ -37,6 +36,8 @@ class MetadataStoreMixin:
         """Get the graphql client."""
         return GraphQLClient(common_configurations.metadata_store_api_base)
 
+    # RECIPE RUN STATUS
+
     def metadata_store_change_recipe_run_to_inprogress(self):
         """Set the recipe run status to "INPROGRESS"."""
         self._metadata_store_change_status(status="INPROGRESS", is_complete=False)
@@ -49,6 +50,76 @@ class MetadataStoreMixin:
         """Set the recipe run status to "TRIALSUCCESS"."""
         self._metadata_store_change_status(status="TRIALSUCCESS", is_complete=False)
 
+    def _metadata_store_recipe_run_status_id(self, status: str) -> None | int:
+        """Find the id of a recipe run status."""
+        params = RecipeRunStatusQuery(recipeRunStatusName=status)
+        response = self.metadata_store_client.execute_gql_query(
+            query_base="recipeRunStatuses",
+            query_response_cls=RecipeRunStatusResponse,
+            query_parameters=params,
+        )
+        if len(response) > 0:
+            return response[0].recipeRunStatusId
+
+    def _metadata_store_create_recipe_run_status(self, status: str, is_complete: bool) -> int:
+        """
+        Add a new recipe run status to the db.
+
+        :param status: name of the status to add
+        :param is_complete: does the new status correspond to an accepted completion state
+        """
+        recipe_run_statuses = {
+            "INPROGRESS": "Recipe run is currently undergoing processing",
+            "COMPLETEDSUCCESSFULLY": "Recipe run processing completed with no errors",
+            "TRIALSUCCESS": "Recipe run trial processing completed with no errors. Recipe run not "
+            "marked complete.",
+        }
+
+        if not isinstance(status, str):
+            raise TypeError(f"status must be of type str: {status}")
+        if not isinstance(is_complete, bool):
+            raise TypeError(f"is_complete must be of type bool: {is_complete}")
+        params = RecipeRunStatusMutation(
+            recipeRunStatusName=status,
+            isComplete=is_complete,
+            recipeRunStatusDescription=recipe_run_statuses[status],
+        )
+        recipe_run_status_response = self.metadata_store_client.execute_gql_mutation(
+            mutation_base="createRecipeRunStatus",
+            mutation_response_cls=RecipeRunStatusResponse,
+            mutation_parameters=params,
+        )
+        return recipe_run_status_response.recipeRunStatus.recipeRunStatusId
+
+    def _metadata_store_change_status(self, status: str, is_complete: bool):
+        """Change the recipe run status of a recipe run to the given status."""
+        recipe_run_status_id = self._metadata_store_recipe_run_status_id(status=status)
+        if not recipe_run_status_id:
+            recipe_run_status_id = self._metadata_store_create_recipe_run_status(
+                status=status, is_complete=is_complete
+            )
+        self._metadata_store_update_status(recipe_run_status_id=recipe_run_status_id)
+
+    def _metadata_store_update_status(
+        self,
+        recipe_run_status_id: int,
+    ):
+        """
+        Change the status of a given recipe run id.
+
+        :param recipe_run_status_id: the new status to use
+        """
+        params = RecipeRunMutation(
+            recipeRunId=self.recipe_run_id, recipeRunStatusId=recipe_run_status_id
+        )
+        self.metadata_store_client.execute_gql_mutation(
+            mutation_base="updateRecipeRun",
+            mutation_parameters=params,
+            mutation_response_cls=RecipeRunMutationResponse,
+        )
+
+    # RECEIPT
+
     def metadata_store_add_dataset_receipt_account(
         self, dataset_id: str, expected_object_count: int
     ):
@@ -62,10 +133,12 @@ class MetadataStoreMixin:
             mutation_response_cls=DatasetCatalogReceiptAccountResponse,
         )
 
+    # PROVENANCE
+
     def metadata_store_record_provenance(self, is_task_manual: bool, library_versions: str):
         """Record the provenance record in the metadata store."""
         params = RecipeRunProvenanceMutation(
-            inputDatasetId=self.metadata_store_input_dataset_id,
+            inputDatasetId=self.metadata_store_recipe_run.recipeInstance.inputDatasetId,
             isTaskManual=is_task_manual,
             recipeRunId=self.recipe_run_id,
             taskName=self.task_name,
@@ -78,6 +151,8 @@ class MetadataStoreMixin:
             mutation_response_cls=RecipeRunProvenanceResponse,
         )
 
+    # QUALITY
+
     def metadata_store_add_quality_data(self, dataset_id: str, quality_data: list[dict]):
         """Add the quality data to the metadata-store."""
         if self.metadata_store_quality_data_exists(dataset_id):
@@ -119,222 +194,60 @@ class MetadataStoreMixin:
         )
         return bool(response)
 
-    def metadata_store_recipe_run_configuration(self) -> dict:
-        """Get the recipe run configuration from the metadata store."""
-        configuration_json = self._metadata_store_recipe_run().configuration
-        if configuration_json is None:
-            return {}
-        try:
-            configuration = json.loads(configuration_json)
-            if not isinstance(configuration, dict):
-                raise ValueError(
-                    f"Invalid recipe run configuration format.  "
-                    f"Expected json encoded dictionary, received json encoded {type(configuration)}"
-                )
-            return configuration
-        except (json.JSONDecodeError, ValueError, TypeError, UnicodeDecodeError) as e:
-            logger.error(f"Invalid recipe run configuration")
-            raise e
+    # INPUT DATASET RECIPE RUN
 
     @cached_property
-    def metadata_store_input_dataset_parts(self) -> list[InputDatasetPartResponse]:
-        """Get the input dataset parts from the metadata store."""
+    def metadata_store_input_dataset_recipe_run_response(self) -> InputDatasetRecipeRunResponse:
+        """Get the input dataset recipe run response from the metadata store."""
         params = RecipeRunQuery(recipeRunId=self.recipe_run_id)
         response = self.metadata_store_client.execute_gql_query(
             query_base="recipeRuns",
             query_response_cls=InputDatasetRecipeRunResponse,
             query_parameters=params,
-        )  # queried independently of other recipe run metadata for performance
-        recipe_run = response[0]
-        return [
-            part_link.inputDatasetPart
-            for part_link in recipe_run.recipeInstance.inputDataset.inputDatasetInputDatasetParts
-        ]
-
-    def _metadata_store_filter_input_dataset_parts(
-        self, input_dataset_part_type_name: str
-    ) -> InputDatasetPartResponse | None:
-        """Filter the input dataset parts based on the input dataset part type name."""
-        target_parts = [
-            part
-            for part in self.metadata_store_input_dataset_parts
-            if part.inputDatasetPartType.inputDatasetPartTypeName == input_dataset_part_type_name
-        ]
-        if not target_parts:
-            return
-        if len(target_parts) == 1:
-            return target_parts[0]
-        raise ValueError(
-            f"Multiple ({len(target_parts)}) input dataset parts found for "
-            f"{input_dataset_part_type_name=}."
-        )
-
-    @property
-    def _metadata_store_input_dataset_observe_frames_part(
-        self,
-    ) -> InputDatasetPartResponse | None:
-        """Get the input dataset part for observe frames."""
-        return self._metadata_store_filter_input_dataset_parts(
-            input_dataset_part_type_name="observe_frames",
-        )
-
-    @property
-    def metadata_store_input_dataset_observe_frames_part_id(self) -> int | None:
-        """Get the input dataset part id for observe frames."""
-        if part := self._metadata_store_input_dataset_observe_frames_part:
-            return part.inputDatasetPartId
-
-    @property
-    def metadata_store_input_dataset_observe_frames_part_document(
-        self,
-    ) -> input_dataset_part_document_type_hint:
-        """Get the input dataset part document for observe frames."""
-        if part := self._metadata_store_input_dataset_observe_frames_part:
-            return part.inputDatasetPartDocument
-
-    @property
-    def _metadata_store_input_dataset_calibration_frames_part(
-        self,
-    ) -> InputDatasetPartResponse | None:
-        """Get the input dataset part for calibration frames."""
-        return self._metadata_store_filter_input_dataset_parts(
-            input_dataset_part_type_name="calibration_frames"
+            response_encoder=InputDatasetRecipeRunResponse.model_validate,
         )
-
-    @property
-    def metadata_store_input_dataset_calibration_frames_part_id(self) -> int | None:
-        """Get the input dataset part id for calibration frames."""
-        if part := self._metadata_store_input_dataset_calibration_frames_part:
-            return part.inputDatasetPartId
-
-    @property
-    def metadata_store_input_dataset_calibration_frames_part_document(
-        self,
-    ) -> input_dataset_part_document_type_hint:
-        """Get the input dataset part document for calibration frames."""
-        if part := self._metadata_store_input_dataset_calibration_frames_part:
-            return part.inputDatasetPartDocument
-
-    @property
-    def _metadata_store_input_dataset_parameters_part(
-        self,
-    ) -> InputDatasetPartResponse | None:
-        """Get the input dataset part for parameters."""
-        return self._metadata_store_filter_input_dataset_parts(
-            input_dataset_part_type_name="parameters"
+        return response[0]
+
+    def _metadata_store_input_dataset_part(
+        self, part_type: Literal["observe_frames", "calibration_frames", "parameters"]
+    ) -> InputDatasetPartResponse:
+        """Get the input dataset part by input dataset part type name."""
+        part_type_dict = {}
+        parts = (
+            self.metadata_store_input_dataset_recipe_run_response.recipeInstance.inputDataset.inputDatasetInputDatasetParts
         )
+        for part in parts:
+            part_type_name = part.inputDatasetPart.inputDatasetPartType.inputDatasetPartTypeName
+            if part_type_name in part_type_dict.keys():
+                raise ValueError(f"Multiple input dataset parts found for {part_type_name=}.")
+            part_type_dict[part_type_name] = part.inputDatasetPart
+        return part_type_dict.get(part_type)
 
     @property
-    def metadata_store_input_dataset_parameters_part_id(self) -> int | None:
-        """Get the input dataset part id for parameters."""
-        if part := self._metadata_store_input_dataset_parameters_part:
-            return part.inputDatasetPartId
+    def metadata_store_input_dataset_observe_frames(self) -> InputDatasetPartResponse:
+        """Get the input dataset part for the observe frames."""
+        return self._metadata_store_input_dataset_part(part_type="observe_frames")
 
     @property
-    def metadata_store_input_dataset_parameters_part_document(
-        self,
-    ) -> input_dataset_part_document_type_hint:
-        """Get the input dataset part document for parameters."""
-        if part := self._metadata_store_input_dataset_parameters_part:
-            return part.inputDatasetPartDocument
+    def metadata_store_input_dataset_calibration_frames(self) -> InputDatasetPartResponse:
+        """Get the input dataset part for the calibration frames."""
+        return self._metadata_store_input_dataset_part(part_type="calibration_frames")
 
     @property
-    def metadata_store_input_dataset_id(self) -> int:
-        """Get the input dataset id from the metadata store."""
-        return self._metadata_store_recipe_run().recipeInstance.inputDatasetId
+    def metadata_store_input_dataset_parameters(self) -> InputDatasetPartResponse:
+        """Get the input dataset part for the parameters."""
+        return self._metadata_store_input_dataset_part(part_type="parameters")
 
-    @property
-    def metadata_store_recipe_instance_id(self) -> int:
-        """Get the recipe instance id from the metadata store."""
-        return self._metadata_store_recipe_run().recipeInstanceId
-
-    @property
-    def metadata_store_recipe_id(self) -> int:
-        """Get the recipe id from the metadata store."""
-        return self._metadata_store_recipe_run().recipeInstance.recipeId
+    # RECIPE RUN
 
-    @property
-    def metadata_store_recipe_run_provenance(self) -> list[RecipeRunProvenanceResponse]:
-        """Get all the provenance records for the recipe run."""
-        return self._metadata_store_recipe_run().recipeRunProvenances
-
-    def _metadata_store_recipe_run(self, allow_cache: bool = True) -> RecipeRunResponse:
-        is_cached = bool(getattr(self, "_recipe_run_cache", False))
-        if is_cached and allow_cache:
-            return self._recipe_run_cache
+    @cached_property
+    def metadata_store_recipe_run(self) -> RecipeRunResponse:
+        """Get the recipe run response from the metadata store."""
         params = RecipeRunQuery(recipeRunId=self.recipe_run_id)
         response = self.metadata_store_client.execute_gql_query(
             query_base="recipeRuns",
             query_response_cls=RecipeRunResponse,
             query_parameters=params,
+            response_encoder=RecipeRunResponse.model_validate,
         )
-        self._recipe_run_cache = response[0]
-        return self._recipe_run_cache
-
-    def _metadata_store_change_status(self, status: str, is_complete: bool):
-        """Change the recipe run status of a recipe run to the given status."""
-        recipe_run_status_id = self._metadata_store_recipe_run_status_id(status=status)
-        if not recipe_run_status_id:
-            recipe_run_status_id = self._metadata_store_create_recipe_run_status(
-                status=status, is_complete=is_complete
-            )
-        self._metadata_store_update_status(recipe_run_status_id=recipe_run_status_id)
-
-    def _metadata_store_recipe_run_status_id(self, status: str) -> None | int:
-        """Find the id of a recipe run status."""
-        params = RecipeRunStatusQuery(recipeRunStatusName=status)
-        response = self.metadata_store_client.execute_gql_query(
-            query_base="recipeRunStatuses",
-            query_response_cls=RecipeRunStatusResponse,
-            query_parameters=params,
-        )
-        if len(response) > 0:
-            return response[0].recipeRunStatusId
-
-    def _metadata_store_create_recipe_run_status(self, status: str, is_complete: bool) -> int:
-        """
-        Add a new recipe run status to the db.
-
-        :param status: name of the status to add
-        :param is_complete: does the new status correspond to an accepted completion state
-        """
-        recipe_run_statuses = {
-            "INPROGRESS": "Recipe run is currently undergoing processing",
-            "COMPLETEDSUCCESSFULLY": "Recipe run processing completed with no errors",
-            "TRIALSUCCESS": "Recipe run trial processing completed with no errors. Recipe run not "
-            "marked complete.",
-        }
-
-        if not isinstance(status, str):
-            raise TypeError(f"status must be of type str: {status}")
-        if not isinstance(is_complete, bool):
-            raise TypeError(f"is_complete must be of type bool: {is_complete}")
-        params = RecipeRunStatusMutation(
-            recipeRunStatusName=status,
-            isComplete=is_complete,
-            recipeRunStatusDescription=recipe_run_statuses[status],
-        )
-        recipe_run_status_response = self.metadata_store_client.execute_gql_mutation(
-            mutation_base="createRecipeRunStatus",
-            mutation_response_cls=RecipeRunStatusResponse,
-            mutation_parameters=params,
-        )
-        return recipe_run_status_response.recipeRunStatus.recipeRunStatusId
-
-    def _metadata_store_update_status(
-        self,
-        recipe_run_status_id: int,
-    ):
-        """
-        Change the status of a given recipe run id.
-
-        :param recipe_run_status_id: the new status to use
-        """
-        params = RecipeRunMutation(
-            recipeRunId=self.recipe_run_id, recipeRunStatusId=recipe_run_status_id
-        )
-        self.metadata_store_client.execute_gql_mutation(
-            mutation_base="updateRecipeRun",
-            mutation_parameters=params,
-            mutation_response_cls=RecipeRunMutationResponse,
-        )
+        return response[0]

dkist_processing_common/tasks/output_data_base.py

@@ -19,7 +19,7 @@ class OutputDataBase(WorkflowTaskBase, ABC):
     @cached_property
     def destination_bucket(self) -> str:
         """Get the destination bucket."""
-        return self.metadata_store_recipe_run_configuration().get("destination_bucket", "data")
+        return self.metadata_store_recipe_run.configuration.get("destination_bucket", "data")
 
     def format_object_key(self, path: Path) -> str:
         """

dkist_processing_common/tasks/teardown.py

@@ -22,7 +22,7 @@ class TeardownBase(WorkflowTaskBase, ABC):
     @property
     def teardown_enabled(self) -> bool:
         """Recipe run configuration indicating if data should be removed at the end of a run."""
-        return self.metadata_store_recipe_run_configuration().get("teardown_enabled", True)
+        return self.metadata_store_recipe_run.configuration.get("teardown_enabled", True)
 
     def run(self) -> None:
         """Run method for Teardown class."""

dkist_processing_common/tasks/transfer_input_data.py

@@ -19,11 +19,11 @@ class TransferL0Data(WorkflowTaskBase, GlobusMixin, InputDatasetMixin):
 
     def download_input_dataset(self):
         """Get the input dataset document parts and save it to scratch with the appropriate tags."""
-        if doc := self.metadata_store_input_dataset_observe_frames_part_document:
+        if doc := self.metadata_store_input_dataset_observe_frames.inputDatasetPartDocument:
             self.write(doc.encode("utf-8"), tags=Tag.input_dataset_observe_frames())
-        if doc := self.metadata_store_input_dataset_calibration_frames_part_document:
+        if doc := self.metadata_store_input_dataset_calibration_frames.inputDatasetPartDocument:
             self.write(doc.encode("utf-8"), tags=Tag.input_dataset_calibration_frames())
-        if doc := self.metadata_store_input_dataset_parameters_part_document:
+        if doc := self.metadata_store_input_dataset_parameters.inputDatasetPartDocument:
             self.write(doc.encode("utf-8"), tags=Tag.input_dataset_parameters())
 
     def format_transfer_items(

dkist_processing_common/tasks/trial_output_data.py

@@ -44,12 +44,12 @@ class TransferTrialData(TransferDataBase, GlobusMixin):
     @cached_property
     def destination_bucket(self) -> str:
         """Get the destination bucket with a trial default."""
-        return self.metadata_store_recipe_run_configuration().get("destination_bucket", "etc")
+        return self.metadata_store_recipe_run.configuration.get("destination_bucket", "etc")
 
     @property
     def destination_root_folder(self) -> Path:
         """Format the destination root folder with a value that can be set in the recipe run configuration."""
-        root_name_from_configuration = self.metadata_store_recipe_run_configuration().get(
+        root_name_from_configuration = self.metadata_store_recipe_run.configuration.get(
             "trial_root_directory_name"
         )
         root_name = Path(root_name_from_configuration or super().destination_root_folder)
@@ -59,9 +59,9 @@ class TransferTrialData(TransferDataBase, GlobusMixin):
     @property
     def destination_folder(self) -> Path:
         """Format the destination folder with a parent that can be set by the recipe run configuration."""
-        dir_name = self.metadata_store_recipe_run_configuration().get(
-            "trial_directory_name"
-        ) or Path(self.constants.dataset_id)
+        dir_name = self.metadata_store_recipe_run.configuration.get("trial_directory_name") or Path(
+            self.constants.dataset_id
+        )
         return self.destination_root_folder / dir_name
 
     @property
@@ -71,7 +71,7 @@ class TransferTrialData(TransferDataBase, GlobusMixin):
         Defaults to transferring all product files.  Setting `trial_exclusive_transfer_tag_lists` in the
         recipe run configuration to a list of tag lists will override the default.
         """
-        return self.metadata_store_recipe_run_configuration().get(
+        return self.metadata_store_recipe_run.configuration.get(
             "trial_exclusive_transfer_tag_lists", self.default_transfer_tag_lists
         )
 

dkist_processing_common/tasks/write_l1.py

@@ -107,17 +107,17 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
     @cached_property
     def tile_size_param(self) -> int:
         """Get the tile size parameter for compression."""
-        return self.metadata_store_recipe_run_configuration().get("tile_size", None)
+        return self.metadata_store_recipe_run.configuration.get("tile_size", None)
 
     @cached_property
     def validate_l1_on_write(self) -> bool:
         """Check for validate on write."""
-        return self.metadata_store_recipe_run_configuration().get("validate_l1_on_write", True)
+        return self.metadata_store_recipe_run.configuration.get("validate_l1_on_write", True)
 
     @cached_property
     def workflow_had_manual_intervention(self):
         """Indicate determining if any provenance capturing steps had manual intervention."""
-        for provenance_record in self.metadata_store_recipe_run_provenance:
+        for provenance_record in self.metadata_store_recipe_run.recipeRunProvenances:
             if provenance_record.isTaskManual:
                 return True
         return False
@@ -201,8 +201,8 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         header["FRAMEVOL"] = -1.0
         header["PROCTYPE"] = "L1"
         header["RRUNID"] = self.recipe_run_id
-        header["RECIPEID"] = self.metadata_store_recipe_id
-        header["RINSTID"] = self.metadata_store_recipe_instance_id
+        header["RECIPEID"] = self.metadata_store_recipe_run.recipeInstance.recipeId
+        header["RINSTID"] = self.metadata_store_recipe_run.recipeInstanceId
         header["EXTNAME"] = "observation"
         header["SOLARNET"] = 1
         header["OBS_HDU"] = 1
@@ -214,10 +214,10 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         header["CADMAX"] = self.constants.maximum_cadence
         header["CADVAR"] = self.constants.variance_cadence
         # Keywords to support reprocessing
-        if ids_par_id := self.metadata_store_input_dataset_parameters_part_id:
+        if ids_par_id := self.metadata_store_input_dataset_parameters.inputDatasetPartId:
             header["IDSPARID"] = ids_par_id
-        header["IDSOBSID"] = self.metadata_store_input_dataset_observe_frames_part_id
-        if ids_cal_id := self.metadata_store_input_dataset_calibration_frames_part_id:
+        header["IDSOBSID"] = self.metadata_store_input_dataset_observe_frames.inputDatasetPartId
+        if ids_cal_id := self.metadata_store_input_dataset_calibration_frames.inputDatasetPartId:
             header["IDSCALID"] = ids_cal_id
         header["WKFLNAME"] = self.workflow_name
         header["WKFLVERS"] = self.workflow_version

dkist_processing_common/tests/conftest.py

@@ -368,7 +368,7 @@ class FakeGQLClient:
                                                     "key1",
                                                     "key2"
                                                 ]
-                                            },
+                                            }
                                         ]""",
                                             inputDatasetPartType=InputDatasetPartTypeResponse(
                                                 inputDatasetPartTypeName="observe_frames"
@@ -385,7 +385,7 @@ class FakeGQLClient:
                                                     "key3",
                                                     "key4"
                                                 ]
-                                            },
+                                            }
                                         ]""",
                                             inputDatasetPartType=InputDatasetPartTypeResponse(
                                                 inputDatasetPartTypeName="calibration_frames"
@@ -420,7 +420,8 @@ class FakeGQLClient:
 class FakeGQLClientNoRecipeConfiguration(FakeGQLClient):
     def execute_gql_query(self, **kwargs):
         response = super().execute_gql_query(**kwargs)
-        response[0].configuration = None
+        if type(response[0]) == RecipeRunResponse:
+            response[0].configuration = {}
         return response
 
 

dkist_processing_common/tests/test_teardown.py

@@ -24,7 +24,7 @@ def make_mock_GQL_with_configuration():
                 response = super().execute_gql_query(**kwargs)
                 if isinstance(response, list):
                     if isinstance(response[0], RecipeRunResponse):
-                        response[0].configuration = json.dumps(configuration)
+                        response[0].configuration = configuration
                 return response
 
         return TeardownFakeGQLClient

dkist_processing_common/tests/test_trial_output_data.py

@@ -29,13 +29,11 @@ def recipe_run_configuration(
             response = super().execute_gql_query(**kwargs)
             if isinstance(response, list):
                 if isinstance(response[0], RecipeRunResponse):
-                    response[0].configuration = json.dumps(
-                        {
-                            "trial_root_directory_name": custom_root_name,
-                            "trial_directory_name": custom_dir_name,
-                            "destination_bucket": destination_bucket,
-                        }
-                    )
+                    response[0].configuration = {
+                        "trial_root_directory_name": custom_root_name,
+                        "trial_directory_name": custom_dir_name,
+                        "destination_bucket": destination_bucket,
+                    }
             return response
 
     return GQLClientWithConfiguration
@@ -50,14 +48,12 @@ def recipe_run_configuration_with_tag_lists(
             response = super().execute_gql_query(**kwargs)
             if isinstance(response, list):
                 if isinstance(response[0], RecipeRunResponse):
-                    response[0].configuration = json.dumps(
-                        {
-                            "trial_root_directory_name": custom_root_name,
-                            "trial_directory_name": custom_dir_name,
-                            "destination_bucket": destination_bucket,
-                            "trial_exclusive_transfer_tag_lists": exclusive_tag_lists,
-                        }
-                    )
+                    response[0].configuration = {
+                        "trial_root_directory_name": custom_root_name,
+                        "trial_directory_name": custom_dir_name,
+                        "destination_bucket": destination_bucket,
+                        "trial_exclusive_transfer_tag_lists": exclusive_tag_lists,
+                    }
             return response
 
     return GQLClientWithConfiguration

dkist_processing_common/tests/test_workflow_task_base.py

@@ -204,7 +204,7 @@ class ProvenanceTask(WorkflowTaskBase):
             libraryVersions=library_versions,
             workflowVersion=self.workflow_version,
         )
-        self.write(data=bytes(json.dumps(asdict(params)), "utf-8"), tags=["TEST_PROVENANCE"])
+        self.write(data=bytes(params.model_dump_json(), "utf-8"), tags=["TEST_PROVENANCE"])
 
 
 @pytest.fixture(scope="function")

dkist_processing_common/tests/test_write_l1.py

@@ -457,9 +457,15 @@ def test_reprocessing_keys(write_l1_task, mocker):
     files = list(task.read(tags=[Tag.frame(), Tag.output()]))
     for file in files:
         header = fits.open(file)[1].header
-        assert header["IDSPARID"] == task.metadata_store_input_dataset_parameters_part_id
-        assert header["IDSOBSID"] == task.metadata_store_input_dataset_observe_frames_part_id
-        assert header["IDSCALID"] == task.metadata_store_input_dataset_calibration_frames_part_id
+        assert header["IDSPARID"] == task.metadata_store_input_dataset_parameters.inputDatasetPartId
+        assert (
+            header["IDSOBSID"]
+            == task.metadata_store_input_dataset_observe_frames.inputDatasetPartId
+        )
+        assert (
+            header["IDSCALID"]
+            == task.metadata_store_input_dataset_calibration_frames.inputDatasetPartId
+        )
         assert header["WKFLNAME"] == task.workflow_name
         assert header["WKFLVERS"] == task.workflow_version
         assert header["PROCTYPE"] == "L1"

{dkist_processing_common-10.6.0rc2.dist-info → dkist_processing_common-10.6.1rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dkist-processing-common
-Version: 10.6.0rc2
+Version: 10.6.1rc1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-10.6.0rc2.dist-info → dkist_processing_common-10.6.1rc1.dist-info}/RECORD

@@ -1,5 +1,6 @@
 changelog/.gitempty,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-changelog/235.feature.rst,sha256=gIY9mL5ji-K5fsOlMrai3nTLxTPmTlH20yrjjhzrT9Q,158
+changelog/236.misc.1.rst,sha256=0agTibMnm1CvRDjoQPuLN_xJmGDzAUTJCbhwrCVe55o,130
+changelog/236.misc.rst,sha256=IVs7M44t2WkDkHz292SjlIeRKLBX_yB7IkXMr5EMP3U,277
 dkist_processing_common/__init__.py,sha256=490Fwm_GgqpwriQlsYfKcLUZNhZ6GkINtJqcYSIEKoU,319
 dkist_processing_common/config.py,sha256=IcpaD_NvHZU-aLlUNOTdRC4V7ADIvVQwrZ2dHhIr4NY,4247
 dkist_processing_common/manual.py,sha256=4sGj3cChdnEpwzq2GSNSbTpUf2oS3YUWiXna6ALfMF4,7037
@@ -23,7 +24,7 @@ dkist_processing_common/models/__init__.py,sha256=6LMqemdzVZ87fRrpAsbEnTtWZ02_Gu
 dkist_processing_common/models/constants.py,sha256=1Eb8RDeuCr6brl237iGKxYLWCH49I6bOUEj_Tv-zFbQ,5441
 dkist_processing_common/models/fits_access.py,sha256=Au9JROwhVla9zb_u0dN8mIWiSJd_Pca0oOr4N1hN0HY,4113
 dkist_processing_common/models/flower_pot.py,sha256=59C5uGYKyMyncqQYxhzDZWl8k1DRZFB6s9RF-HFp9mY,5128
-dkist_processing_common/models/graphql.py,sha256=b8kI-Pq12gixOtotCWHVTfEBT7XKBlvV_IaCM9qkKvg,3992
+dkist_processing_common/models/graphql.py,sha256=GBCRk6o9TGVCNy7hc9Yrmn5YZW76JBs9AY1Q5muvG1Y,4040
 dkist_processing_common/models/message.py,sha256=DRW7Qhl01dF5KagcqLta5U-uzdOMewrsHvMatDT6jnk,1684
 dkist_processing_common/models/message_queue_binding.py,sha256=ROQ2ZQE3TCr4gVbz4WggvUSExAiWP8SD_GjjQl482M8,1012
 dkist_processing_common/models/metric_code.py,sha256=McXAEF1Sa0_YlR1niXYLJWLFHhdLQhmYw9Xtpr5FGws,815
@@ -52,25 +53,25 @@ dkist_processing_common/tasks/__init__.py,sha256=uH8DTiQP-cx4vMK53S4LYGZGmbip5s0
 dkist_processing_common/tasks/assemble_movie.py,sha256=9K4sgXyRKaX7UsFBIs138pG3AtClwLLopYw3ZQY3ok4,12771
 dkist_processing_common/tasks/base.py,sha256=k_IJR5sVV6ennX0sbeb0C6dciqshdY7CKjtWHy_adm8,13143
 dkist_processing_common/tasks/l1_output_data.py,sha256=IM-nvGaTM5r-z-9vHr2wovPVUpuNCah-cWIFMO2fcII,10576
-dkist_processing_common/tasks/output_data_base.py,sha256=sbvTTBgqGvmsGmu3Svl4RSR1WSWejKdsF5_mhO8aRBo,3699
+dkist_processing_common/tasks/output_data_base.py,sha256=hxhO06rAsJkPsuG-XyWa4PvCrFKrseIek2AOQPJSn5M,3697
 dkist_processing_common/tasks/parse_l0_input_data.py,sha256=iRMGdvhxBobNsTDQ0IEl0myDfB4P_xpxA00guuBWDj8,7986
 dkist_processing_common/tasks/quality_metrics.py,sha256=g6MUq8s8jELDinkn6o45rfONyODw92JyVMrzb7Dd7OI,12458
-dkist_processing_common/tasks/teardown.py,sha256=TNghJIBhLucxJsabuygDbU4oL0RqdTwq8DqOcP5tub8,2370
-dkist_processing_common/tasks/transfer_input_data.py,sha256=oR1Y-EEodH8uZT26j-7PfQ9uIMp96GOUemmLSCCb67w,5212
+dkist_processing_common/tasks/teardown.py,sha256=zKMA5oZDs_p6Dr7IuMtmRgmHbeBPYkx1dv_-RPuChqY,2368
+dkist_processing_common/tasks/transfer_input_data.py,sha256=Sc3wBh48F1LhghRZyR3E39NAnMGExI7liibCfgu3FcE,5245
 dkist_processing_common/tasks/trial_catalog.py,sha256=Y3DKstRfMS8nWWtJFMB0MUVPlZ1jWS_2jhJGMWwxy50,8748
-dkist_processing_common/tasks/trial_output_data.py,sha256=vE4nRdY3RP19MuKKPl9JKYYi81DtMbs3VghAZoVJ_2s,6890
-dkist_processing_common/tasks/write_l1.py,sha256=KJmfgOiH2R8-lXMIDqNqGkHFMnzAeb6JgF0m3zhj9wI,22375
+dkist_processing_common/tasks/trial_output_data.py,sha256=HHORFHSEN3nGfGB5fIh5qeoAqyousMYTa_g4_R2uzEA,6882
+dkist_processing_common/tasks/write_l1.py,sha256=Nh6d7pSRtvwEvlL5fyidytX4lYiwUEveUZPDxOpclzE,22448
 dkist_processing_common/tasks/mixin/__init__.py,sha256=-g-DQbU7m1bclJYuFe3Yh757V-35GIDTbstardKQ7nU,68
 dkist_processing_common/tasks/mixin/globus.py,sha256=QAV8VElxMAqxJ2KSB_bJaraceovYfjHXjOdocrTCkIA,6592
 dkist_processing_common/tasks/mixin/input_dataset.py,sha256=dkW5vf_QPgWedHO_Lf9GjBxr1QrUCKs6gIXufUTi7GE,6813
 dkist_processing_common/tasks/mixin/interservice_bus.py,sha256=I7BUh0o8AEX-FZv7gxCts6is0uq9lycWjtTB2KqwBrU,1080
-dkist_processing_common/tasks/mixin/metadata_store.py,sha256=VmcKYnA0RBhhKZadpmUGtVciSMpIkKoMU81IR_SnB04,15619
+dkist_processing_common/tasks/mixin/metadata_store.py,sha256=x77Kpdy7C59ZJnm7TAWsv8_JjWCb_SRylhdgx94zJsU,11846
 dkist_processing_common/tasks/mixin/object_store.py,sha256=Vn4l2XuCimii9Fc3gM-pQGIkTKMv_ldqljlxkLesZLU,3236
 dkist_processing_common/tasks/mixin/quality/__init__.py,sha256=Bgu-DHW7yXLiehglldOCWluEkAP5qh0Hp1F30rh5NFw,383
 dkist_processing_common/tasks/mixin/quality/_base.py,sha256=nZ9IC-O-hsLXa5-tk29B13CZyQIdhJCv0eO9cdkAhWc,8303
 dkist_processing_common/tasks/mixin/quality/_metrics.py,sha256=WenTfa12guIUfm0GzkrK2gduKaOHs03e6RhE6j37Les,54304
 dkist_processing_common/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dkist_processing_common/tests/conftest.py,sha256=0MN62OuP26cT48Xsrj2wOZ2MT66SaFHhj8RwShjjrkc,29272
+dkist_processing_common/tests/conftest.py,sha256=ftvyKNSheU1wKXPsuX290tnMHKMhla2U4e4cgLnRa5k,29323
 dkist_processing_common/tests/test_assemble_movie.py,sha256=XY_ruXSYP5k6s2gUAwlFdnhJ81eyWLSd2O9IkX4RXeo,4165
 dkist_processing_common/tests/test_assemble_quality.py,sha256=fWSHK4UdVqgNjvxQuD40NBUnXrtmthUP7PUbISPV4MQ,16897
 dkist_processing_common/tests/test_base.py,sha256=4ST3__jEHitEQaQs9-0OcqtyEJfIjZsk_6PRYZFV2-U,7124
@@ -94,13 +95,13 @@ dkist_processing_common/tests/test_submit_dataset_metadata.py,sha256=F1IKBFWhjjM
 dkist_processing_common/tests/test_tags.py,sha256=UwlOJ45rkvbfbd5L5m5YltvOxQc8kGqJEn5V81H33U8,5023
 dkist_processing_common/tests/test_task_name.py,sha256=kqFr59XX2K87xzfTlClzDV4-Je1dx72LvdaJ22UE8UU,1233
 dkist_processing_common/tests/test_task_parsing.py,sha256=QXt1X6DTO3_liBD2c-t84DToLeEn7B3J-eteIyN4HEM,4027
-dkist_processing_common/tests/test_teardown.py,sha256=miAMXG5c3wZxFfwBke8F7PR_Ke8KZmbegRKK9oB0tvU,5432
+dkist_processing_common/tests/test_teardown.py,sha256=bw_MTJ5G9fcpKP7z_hW1sE7geZ7pxUEn-9p0trizqYY,5420
 dkist_processing_common/tests/test_transfer_input_data.py,sha256=itmOFrK7x4G26cj0Xkf015Xn09G9YDdJunXybGgD8Ek,6658
 dkist_processing_common/tests/test_transfer_l1_output_data.py,sha256=27PifkyH3RZg0nsM-AjmrFJ-hbYuCk5Tt_0Zx8PJBfM,2109
 dkist_processing_common/tests/test_trial_catalog.py,sha256=YuZsGlm5WE_XClaqzBB9U-EzQF4fdju8ocrYY-MeXEk,6844
-dkist_processing_common/tests/test_trial_output_data.py,sha256=rXrf2oncBIytht3yKqA3yCDsqDLoYanD9_0H5GwXTzc,12900
-dkist_processing_common/tests/test_workflow_task_base.py,sha256=EVxlz_qOxFn2AdzpppLcW7HTcBj_QC4UPouGGfWHls4,10519
-dkist_processing_common/tests/test_write_l1.py,sha256=NmLhi9iyXf02ry_y1eavPjlD73EDiX1ZBUk4Zy2vqgQ,20246
+dkist_processing_common/tests/test_trial_output_data.py,sha256=5tIn7lQ0L5oshsJvkZroOt-Cl02bigDl4DXNNfc5wFI,12748
+dkist_processing_common/tests/test_workflow_task_base.py,sha256=Z5aPW5LQtS0UWJiYho4X0r-2gPLfzpkmMwfmaoFLjMg,10517
+dkist_processing_common/tests/test_write_l1.py,sha256=Wr8UqYYxQk2N2mkITizFWz9W6VJjcuVom7vlpSuw3is,20351
 docs/Makefile,sha256=qnlVz6PuBqE39NfHWuUnHhNEA-EFgT2-WJNNNy9ttfk,4598
 docs/changelog.rst,sha256=S2jPASsWlQxSlAPqdvNrYvhk9k3FcFWNXFNDYXBSjl4,120
 docs/conf.py,sha256=FkX575cqTqZGCcLAjg2MlvE8Buj1Vt3CpHNgZxG256E,1890
@@ -109,7 +110,7 @@ docs/landing_page.rst,sha256=aPAuXFhBx73lEZ59B6E6JXxkK0LlxzD0n-HXqHrfumQ,746
 docs/make.bat,sha256=mBAhtURwhQ7yc95pqwJzlhqBSvRknr1aqZ5s8NKvdKs,4513
 docs/requirements.txt,sha256=Kbl_X4c7RQZw035YTeNB63We6I7pvXFU4T0Uflp2yDY,29
 licenses/LICENSE.rst,sha256=piZaQplkzOMmH1NXg6QIdo9wwo9pPCoHkvm2-DmH76E,1462
-dkist_processing_common-10.6.0rc2.dist-info/METADATA,sha256=SFlO2ZOzBfCo8zqDHAfIIthsqoX9EIDrhg4Czajxd0Y,6963
-dkist_processing_common-10.6.0rc2.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-dkist_processing_common-10.6.0rc2.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
-dkist_processing_common-10.6.0rc2.dist-info/RECORD,,
+dkist_processing_common-10.6.1rc1.dist-info/METADATA,sha256=kc9wibJ7S38-AQeFbCGDXEQ00nkwvjOU7ITGUdWNK_E,6963
+dkist_processing_common-10.6.1rc1.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+dkist_processing_common-10.6.1rc1.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
+dkist_processing_common-10.6.1rc1.dist-info/RECORD,,

changelog/235.feature.rst

@@ -1,2 +0,0 @@
-Add the `RetarderNameBud` that can parse the name of the GOS retarder and ensure that only a single retarder was used
-for the given set of POLCAL input data.