dkist-processing-common 11.7.0rc6__py3-none-any.whl → 11.9.1__py3-none-any.whl

dkist_processing_common/config.py

@@ -3,12 +3,20 @@
 from dkist_processing_core.config import DKISTProcessingCoreConfiguration
 from dkist_service_configuration.settings import DEFAULT_MESH_SERVICE
 from dkist_service_configuration.settings import MeshService
+from pydantic import BaseModel
 from pydantic import Field
 from talus import ConnectionRetryerFactory
 from talus import ConsumerConnectionParameterFactory
 from talus import ProducerConnectionParameterFactory
 
 
+class GlobusClientCredential(BaseModel):
+    """Globus client credential."""
+
+    client_id: str = Field(..., description="Globus client ID for transfers.")
+    client_secret: str = Field(..., description="Globus client secret for transfers.")
+
+
 class DKISTProcessingCommonConfiguration(DKISTProcessingCoreConfiguration):
     """Common configurations."""
 
@@ -40,14 +48,28 @@ class DKISTProcessingCommonConfiguration(DKISTProcessingCoreConfiguration):
         default=None, description="S3 download configuration for the object store."
     )
     # globus
-    globus_transport_params: dict = Field(
-        default_factory=dict, description="Globus transfer parameters."
+    globus_max_retries: int = Field(
+        default=5, description="Max retries for transient errors on calls to the globus api."
     )
-    globus_client_id: str | None = Field(
-        default=None, description="Globus client ID for inbound/outbound transfers."
+    globus_inbound_client_credentials: list[GlobusClientCredential] = Field(
+        default_factory=list,
+        description="Globus client credentials for inbound transfers.",
+        examples=[
+            [
+                {"client_id": "id1", "client_secret": "secret1"},
+                {"client_id": "id2", "client_secret": "secret2"},
+            ],
+        ],
     )
-    globus_client_secret: str | None = Field(
-        default=None, description="Globus client secret for inbound/outbound transfers."
+    globus_outbound_client_credentials: list[GlobusClientCredential] = Field(
+        default_factory=list,
+        description="Globus client credentials for outbound transfers.",
+        examples=[
+            [
+                {"client_id": "id3", "client_secret": "secret3"},
+                {"client_id": "id4", "client_secret": "secret4"},
+            ],
+        ],
     )
     object_store_endpoint: str | None = Field(
         default=None, description="Object store Globus Endpoint ID."

dkist_processing_common/models/graphql.py

@@ -176,36 +176,3 @@ class RecipeRunProvenanceMutation(GraphqlBaseModel):
     libraryVersions: str
     workflowVersion: str
     codeVersion: str | None = None
-
-
-class QualityCreation(GraphqlBaseModel):
-    """Quality data creation record."""
-
-    datasetId: str
-    metricCode: str
-    facet: str | None = None
-    name: str | None = None
-    description: str | None = None
-    statement: str | None = None
-    # JSON array
-    warnings: str | None = None
-    # JSON objects
-    plotData: str | None = None
-    multiPlotData: str | None = None
-    tableData: str | None = None
-    histogramData: str | None = None
-    modmatData: str | None = None
-    raincloudData: str | None = None
-    efficiencyData: str | None = None
-
-
-class QualitiesRequest(GraphqlBaseModel):
-    """Query parameters for quality data."""
-
-    datasetId: str
-
-
-class QualityResponse(GraphqlBaseModel):
-    """Query Response for quality data."""
-
-    qualityId: int

dkist_processing_common/tasks/l1_output_data.py

@@ -38,8 +38,9 @@ class L1OutputDataBase(OutputDataBase, ABC):
 
     @property
     def dataset_has_quality_data(self) -> bool:
-        """Return True if quality data has been persisted to the metadata-store."""
-        return self.metadata_store_quality_data_exists(dataset_id=self.constants.dataset_id)
+        """Return True if the dataset has quality data."""
+        path_count = self.count(tags=[Tag.output(), Tag.quality_data()])
+        return path_count > 0
 
     def rollback(self):
         """Warn that the metadata-store and the interservice bus retain the effect of this tasks execution.  Rolling back this task may not be achievable without other action."""
@@ -58,6 +59,9 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
             # Movie needs to be transferred separately as the movie headers need to go with it
             self.transfer_movie()
 
+        with self.telemetry_span("Upload quality data"):
+            self.transfer_quality_data()
+
         with self.telemetry_span("Upload science frames"):
             self.transfer_output_frames()
 
@@ -101,6 +105,33 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
             content_type="video/mp4",
         )
 
+    def transfer_quality_data(self):
+        """Transfer quality data to the object store."""
+        paths = list(self.read(tags=[Tag.output(), Tag.quality_data()]))
+        if len(paths) == 0:
+            logger.info(
+                f"No quality data found to upload for dataset. recipe_run_id={self.recipe_run_id}"
+            )
+            return
+
+        if count := len(paths) > 1:
+            # dataset inventory does not support multiple quality data object keys
+            raise RuntimeError(
+                f"Found multiple quality data files to upload.  Not supported."
+                f"{count=}, recipe_run_id={self.recipe_run_id}"
+            )
+
+        with self.telemetry_span(f"Uploading the trial quality data"):
+            path = paths[0]
+            logger.info(f"Uploading quality data: recipe_run_id={self.recipe_run_id}, {path=}")
+            quality_data_object_key = self.format_object_key(path)
+            self.object_store_upload_quality_data(
+                quality_data=path,
+                bucket=self.destination_bucket,
+                object_key=quality_data_object_key,
+                content_type="application/json",
+            )
+
 
 class AssembleQualityData(L1OutputDataBase, QualityMixin):
     """
@@ -128,7 +159,7 @@ class AssembleQualityData(L1OutputDataBase, QualityMixin):
         ):
             self.write(
                 quality_data,
-                tags=Tag.quality_data(),
+                tags=[Tag.output(), Tag.quality_data()],
                 encoder=quality_data_encoder,
                 relative_path=f"{self.constants.dataset_id}_quality_data.json",
             )
@@ -136,31 +167,18 @@ class AssembleQualityData(L1OutputDataBase, QualityMixin):
 
 class SubmitDatasetMetadata(L1OutputDataBase):
     """
-    Add quality data and receipt account to the metadata store.
+    Add receipt account to the metadata store.
 
-    Add the quality data to the Quality database.
     Add a Dataset Receipt Account record to Processing Support for use by the Dataset Catalog Locker.
-    Adds the number of files created during the calibration processing to the Processing Support table
+    Adds the number of files to be created during the calibration processing to the Processing Support table
     for use by the Dataset Catalog Locker.
     """
 
     def run(self) -> None:
         """Run method for this task."""
-        with self.telemetry_span(f"Storing quality data to metadata store"):
-            # each quality_data file is a list - this will combine the elements of multiple lists into a single list
-            quality_data = list(
-                chain.from_iterable(
-                    self.read(tags=Tag.quality_data(), decoder=quality_data_decoder)
-                )
-            )
-            self.metadata_store_add_quality_data(
-                dataset_id=self.constants.dataset_id, quality_data=quality_data
-            )
         with self.telemetry_span("Count Expected Outputs"):
             dataset_id = self.constants.dataset_id
             expected_object_count = self.count(tags=Tag.output())
-            if quality_data:
-                expected_object_count += 1
         logger.info(
             f"Adding Dataset Receipt Account: "
             f"{dataset_id=}, {expected_object_count=}, recipe_run_id={self.recipe_run_id}"
@@ -230,24 +248,12 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         messages = [CatalogObjectMessage(body=body) for body in message_bodies]
         return messages
 
-    @property
-    def quality_report_message(self) -> CreateQualityReportMessage:
-        """Create the Quality Report Message."""
-        file_name = Path(f"{self.constants.dataset_id}_quality_report.pdf")
-        body = CreateQualityReportMessageBody(
-            bucket=self.destination_bucket,
-            objectName=self.format_object_key(file_name),
-            conversationId=str(self.recipe_run_id),
-            datasetId=self.constants.dataset_id,
-            incrementDatasetCatalogReceiptCount=True,
-        )
-        return CreateQualityReportMessage(body=body)
-
     def run(self) -> None:
         """Run method for this task."""
         with self.telemetry_span("Gather output data"):
             frames = self.read(tags=self.output_frame_tags)
             movies = self.read(tags=[Tag.output(), Tag.movie()])
+            quality_data = self.read(tags=[Tag.output(), Tag.quality_data()])
         with self.telemetry_span("Create message objects"):
             messages = []
             messages += self.frame_messages(paths=frames)
@@ -256,7 +262,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
             object_message_count = len(messages) - frame_message_count
             dataset_has_quality_data = self.dataset_has_quality_data
             if dataset_has_quality_data:
-                messages.append(self.quality_report_message)
+                messages += self.object_messages(paths=quality_data, object_type="QDATA")
         with self.telemetry_span(
             f"Publish messages: {frame_message_count = }, {object_message_count = }, {dataset_has_quality_data = }"
         ):

dkist_processing_common/tasks/mixin/globus.py

@@ -9,6 +9,8 @@ from globus_sdk import ConfidentialAppAuthClient
 from globus_sdk import GlobusError
 from globus_sdk import TransferClient
 from globus_sdk import TransferData
+from globus_sdk.scopes import TransferScopes
+from globus_sdk.transport import RetryConfig
 
 from dkist_processing_common.config import common_configurations
 
@@ -31,27 +33,32 @@ class GlobusTransferItem:
 class GlobusMixin:
     """Mixin to add methods to a Task to support globus transfers."""
 
-    @property
-    def globus_transfer_client(self) -> TransferClient:
-        """Get the globus transfer client, creating it if it doesn't exist."""
-        if getattr(self, "_globus_transfer_client", False):
-            return self._globus_transfer_client
+    def globus_transfer_client_factory(self, transfer_data: TransferData) -> TransferClient:
+        """Create a globus transfer client based on the direction of transfer and round-robin the available application credentials."""
+        if (
+            transfer_data["source_endpoint"] == common_configurations.object_store_endpoint
+        ):  # inbound
+            client_credentials = common_configurations.globus_inbound_client_credentials
+        else:  # outbound
+            client_credentials = common_configurations.globus_outbound_client_credentials
+
+        # Round-robin the client credentials based on the recipe run id
+        index = self.recipe_run_id % len(client_credentials)
+        selected_credential = client_credentials[index]
+
         confidential_client = ConfidentialAppAuthClient(
-            client_id=common_configurations.globus_client_id,
-            client_secret=common_configurations.globus_client_secret,
-            transport_params=common_configurations.globus_transport_params,
+            client_id=selected_credential.client_id,
+            client_secret=selected_credential.client_secret,
         )
-        authorizer = ClientCredentialsAuthorizer(
-            confidential_client, scopes="urn:globus:auth:scope:transfer.api.globus.org:all"
-        )
-        self._globus_transfer_client = TransferClient(authorizer=authorizer)
-        return self._globus_transfer_client
+        authorizer = ClientCredentialsAuthorizer(confidential_client, scopes=TransferScopes)
+        retry_config = RetryConfig(max_retries=common_configurations.globus_max_retries)
+
+        return TransferClient(authorizer=authorizer, retry_config=retry_config)
 
     def globus_transfer_scratch_to_object_store(
         self,
         transfer_items: list[GlobusTransferItem],
         label: str = None,
-        sync_level: str = None,
         verify_checksum: bool = True,
     ) -> None:
         """Transfer data from scratch to the object store."""
@@ -60,7 +67,6 @@ class GlobusMixin:
             destination_endpoint=common_configurations.object_store_endpoint,
             transfer_items=transfer_items,
             label=label,
-            sync_level=sync_level,
             verify_checksum=verify_checksum,
         )
 
@@ -68,7 +74,6 @@ class GlobusMixin:
         self,
         transfer_items: list[GlobusTransferItem],
         label: str = None,
-        sync_level: str = None,
         verify_checksum: bool = True,
     ) -> None:
         """Transfer data from the object store to scratch."""
@@ -77,7 +82,6 @@ class GlobusMixin:
             destination_endpoint=common_configurations.scratch_endpoint,
             transfer_items=transfer_items,
             label=label,
-            sync_level=sync_level,
             verify_checksum=verify_checksum,
         )
 
@@ -87,7 +91,6 @@ class GlobusMixin:
         destination_endpoint: str,
         transfer_items: list[GlobusTransferItem],
         label: str = None,
-        sync_level: str = None,
         verify_checksum: bool = True,
     ) -> TransferData:
         """Format a globus TransferData instance."""
@@ -95,7 +98,6 @@ class GlobusMixin:
             source_endpoint=source_endpoint,
             destination_endpoint=destination_endpoint,
             label=label,
-            sync_level=sync_level,
             verify_checksum=verify_checksum,
         )
         for item in transfer_items:
@@ -112,7 +114,6 @@ class GlobusMixin:
         destination_endpoint: str,
         transfer_items: list[GlobusTransferItem],
         label: str = None,
-        sync_level: str = None,
         verify_checksum: bool = True,
     ) -> None:
         """Perform a transfer of data using globus."""
@@ -121,7 +122,6 @@ class GlobusMixin:
             destination_endpoint=destination_endpoint,
             transfer_items=transfer_items,
             label=label,
-            sync_level=sync_level,
             verify_checksum=verify_checksum,
         )
         self._blocking_globus_transfer(transfer_data=transfer_data)
@@ -131,24 +131,21 @@ class GlobusMixin:
         source_endpoint: str,
         destination_endpoint: str,
         label: str = None,
-        sync_level: str = None,
         verify_checksum: bool = True,
     ) -> TransferData:
         label = label or "Data Processing Transfer"
         return TransferData(
-            transfer_client=self.globus_transfer_client,
             source_endpoint=source_endpoint,
             destination_endpoint=destination_endpoint,
             label=label,
-            sync_level=sync_level,
             verify_checksum=verify_checksum,
         )
 
     def _blocking_globus_transfer(self, transfer_data: TransferData) -> None:
-        tc = self.globus_transfer_client
-        logger.info(f"Starting globus transfer: label={transfer_data.get('label')}")
+        tc = self.globus_transfer_client_factory(transfer_data=transfer_data)
         transfer_result = tc.submit_transfer(transfer_data)
         task_id = transfer_result["task_id"]
+        logger.info(f"Starting globus transfer: label={transfer_data.get('label')}, {task_id=}, ")
         polling_interval = 60
         while not tc.task_wait(
             task_id=task_id, timeout=polling_interval, polling_interval=polling_interval

dkist_processing_common/tasks/mixin/metadata_store.py

@@ -1,6 +1,5 @@
 """Mixin for a WorkflowDataTaskBase subclass which implements Metadata Store data access functionality."""
 
-import json
 import logging
 from functools import cached_property
 from typing import Literal
@@ -8,15 +7,11 @@ from typing import Literal
 from pydantic import validate_call
 
 from dkist_processing_common._util.graphql import GraphQLClient
-from dkist_processing_common.codecs.quality import QualityDataEncoder
 from dkist_processing_common.config import common_configurations
 from dkist_processing_common.models.graphql import DatasetCatalogReceiptAccountMutation
 from dkist_processing_common.models.graphql import DatasetCatalogReceiptAccountResponse
 from dkist_processing_common.models.graphql import InputDatasetPartResponse
 from dkist_processing_common.models.graphql import InputDatasetRecipeRunResponse
-from dkist_processing_common.models.graphql import QualitiesRequest
-from dkist_processing_common.models.graphql import QualityCreation
-from dkist_processing_common.models.graphql import QualityResponse
 from dkist_processing_common.models.graphql import RecipeRunMutation
 from dkist_processing_common.models.graphql import RecipeRunMutationResponse
 from dkist_processing_common.models.graphql import RecipeRunProvenanceMutation
@@ -150,50 +145,6 @@ class MetadataStoreMixin:
             mutation_response_cls=RecipeRunProvenanceResponse,
         )
 
-    # QUALITY
-
-    def metadata_store_add_quality_data(self, dataset_id: str, quality_data: list[dict]):
-        """Add the quality data to the metadata-store."""
-        if self.metadata_store_quality_data_exists(dataset_id):
-            raise RuntimeError(f"Quality data already persisted for dataset {dataset_id!r}")
-        for metric in quality_data:
-            if (metric_code := metric.get("metric_code")) is None:
-                name = metric.get("name")
-                raise ValueError(f"No metric_code for {name!r} in dataset {dataset_id!r}")
-            params = QualityCreation(
-                datasetId=dataset_id,
-                metricCode=metric_code,
-                facet=metric.get("facet"),
-                name=metric.get("name"),
-                description=metric.get("description"),
-                statement=metric.get("statement"),
-                # JSON array
-                warnings=json.dumps(metric.get("warnings")),
-                # JSON objects
-                plotData=json.dumps(metric.get("plot_data"), cls=QualityDataEncoder),
-                multiPlotData=json.dumps(metric.get("multi_plot_data"), cls=QualityDataEncoder),
-                tableData=json.dumps(metric.get("table_data"), cls=QualityDataEncoder),
-                histogramData=json.dumps(metric.get("histogram_data"), cls=QualityDataEncoder),
-                modmatData=json.dumps(metric.get("modmat_data"), cls=QualityDataEncoder),
-                raincloudData=json.dumps(metric.get("raincloud_data"), cls=QualityDataEncoder),
-                efficiencyData=json.dumps(metric.get("efficiency_data"), cls=QualityDataEncoder),
-            )
-            self.metadata_store_client.execute_gql_mutation(
-                mutation_base="createQuality",
-                mutation_parameters=params,
-                mutation_response_cls=QualityResponse,
-            )
-
-    def metadata_store_quality_data_exists(self, dataset_id: str) -> bool:
-        """Return True if quality data exists in the metadata-store for the given dataset id."""
-        params = QualitiesRequest(datasetId=dataset_id)
-        response = self.metadata_store_client.execute_gql_query(
-            query_base="qualities",
-            query_response_cls=QualityResponse,
-            query_parameters=params,
-        )
-        return bool(response)
-
     # INPUT DATASET RECIPE RUN
 
     @cached_property

dkist_processing_common/tasks/mixin/object_store.py

@@ -55,6 +55,27 @@ class ObjectStoreMixin:
             },
         )
 
+    def object_store_upload_quality_data(
+        self,
+        quality_data: Path | bytes,
+        bucket: str,
+        object_key: str,
+        content_type: str = "application/json",
+    ):
+        """Upload quality data to the object store."""
+        self.object_store_client.upload_object(
+            object_data=quality_data,
+            bucket=bucket,
+            object_key=object_key,
+            verify_checksum=True,
+            content_type=content_type,
+            metadata={
+                "groupname": "DATASET",
+                "groupid": self.constants.dataset_id,
+                "objecttype": "QDATA",
+            },
+        )
+
     def object_store_remove_folder_objects(self, bucket: str, path: Path | str) -> list[str]:
         """
         Remove folder objects (end with /) in the specified bucket and path.

dkist_processing_common/tasks/mixin/quality/_metrics.py

@@ -1356,15 +1356,13 @@ class _WavecalQualityMixin:
         Note that the residuals are the *unweighed* residuals.
         """
         weight_data = np.ones(input_wavelength.size) if weights is None else weights
-        prepared_weights = np.sqrt(weight_data / np.sum(weight_data))
+        prepared_weights = fit_result.prepared_weights
         residuals = fit_result.minimizer_result.residual / prepared_weights
         residuals[~np.isfinite(residuals)] = 0.0
-        best_fit_atlas = input_spectrum - residuals
         normalized_residuals = residuals / input_spectrum
 
-        best_fit_header = fit_result.wavelength_parameters.to_header(axis_num=1)
-        wcs = WCS(best_fit_header)
-        best_fit_wavelength = wcs.spectral.pixel_to_world(np.arange(input_spectrum.size))
+        best_fit_atlas = fit_result.best_fit_atlas
+        best_fit_wavelength = fit_result.best_fit_wavelength_vector
 
         finite_idx = (
             np.isfinite(input_wavelength)
@@ -1378,7 +1376,7 @@ class _WavecalQualityMixin:
         data = {
             "input_wavelength_nm": input_wavelength.to_value(u.nm)[finite_idx].tolist(),
             "input_spectrum": input_spectrum[finite_idx].tolist(),
-            "best_fit_wavelength_nm": best_fit_wavelength.to_value(u.nm)[finite_idx].tolist(),
+            "best_fit_wavelength_nm": best_fit_wavelength[finite_idx].tolist(),
             "best_fit_atlas": best_fit_atlas[finite_idx].tolist(),
             "normalized_residuals": normalized_residuals[finite_idx].tolist(),
             "weights": None if weights is None else weight_data[finite_idx].tolist(),

dkist_processing_common/tasks/trial_catalog.py

@@ -5,15 +5,18 @@ import logging
 from datetime import datetime
 from itertools import chain
 from pathlib import Path
+from typing import Any
 from typing import Generator
 from uuid import uuid4
 
 from dkist_processing_common.codecs.asdf import asdf_fileobj_encoder
+from dkist_processing_common.codecs.basemodel import basemodel_decoder
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.json import json_encoder
 from dkist_processing_common.codecs.path import path_decoder
 from dkist_processing_common.codecs.quality import quality_data_decoder
 from dkist_processing_common.models.fits_access import FitsAccessBase
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.output_data_base import OutputDataBase
 
@@ -27,6 +30,7 @@ INVENTORY_EXTRA_INSTALLED = False
 try:
     from dkist_inventory.inventory import generate_asdf_filename
     from dkist_inventory.inventory import generate_inventory_from_frame_inventory
+    from dkist_inventory.inventory import generate_quality_report_filename
 
     INVENTORY_EXTRA_INSTALLED = True
 except ModuleNotFoundError:
@@ -138,11 +142,15 @@ class CreateTrialAsdf(OutputDataBase):
 
     def run(self) -> None:
         """Generate an ASDF file simulating the ASDF file that would be produced when cataloging the dataset."""
+        with self.telemetry_span("Collate input dataset parameters"):
+            parameters = self.parse_input_dataset_parameters()
+
         with self.telemetry_span("Generate ASDF tree"):
             tree = asdf_tree_from_filenames(
                 filenames=self.absolute_output_frame_paths,
                 hdu=1,  # compressed
                 relative_to=self.scratch.workflow_base_path,
+                parameters=parameters,
             )
 
         trial_history = [
@@ -169,6 +177,37 @@ class CreateTrialAsdf(OutputDataBase):
                     ),
                 )
 
+    def parse_input_dataset_parameters(self) -> list[dict[str, Any]]:
+        """
+        Return the parameters associated with the dataset.
+
+        Returns
+        -------
+        list[dict[str, Any]]
+            A list of dictionaries, each containing a parameter name and its values.
+
+        Raises
+        ------
+        ValueError
+            If there is not exactly one ``InputDatasetPartDocumentList`` found.
+        """
+        part_docs_iter = self.read(
+            tags=Tag.input_dataset_parameters(),
+            decoder=basemodel_decoder,
+            model=InputDatasetPartDocumentList,
+        )
+        docs = list(part_docs_iter)
+
+        if not docs:
+            logger.warning("No parameter list decoded from files")
+            return []
+
+        if len(docs) > 1:
+            raise ValueError(f"Expected 1 parameter list, found {len(docs)}")
+
+        parameters = docs[0].model_dump(by_alias=True).get("doc_list", [])
+        return parameters
+
 
 class CreateTrialQualityReport(OutputDataBase):
     """
@@ -186,6 +225,13 @@ class CreateTrialQualityReport(OutputDataBase):
                 f" but the required dependencies were not found."
             )
 
+        if not INVENTORY_EXTRA_INSTALLED:
+            raise ModuleNotFoundError(
+                f"{self.__class__.__name__} Task requires the dkist-inventory package "
+                f"(e.g. via an 'inventory' pip_extra on dkist_processing_core.Workflow().add_node())"
+                f" but the required dependencies were not found."
+            )
+
     def run(self) -> None:
         """Generate the quality report for the dataset."""
         self.create_trial_quality_report()
@@ -207,5 +253,7 @@ class CreateTrialQualityReport(OutputDataBase):
             self.write(
                 quality_report,
                 tags=[Tag.output(), Tag.quality_report()],
-                relative_path=f"{self.constants.dataset_id}_quality_report.pdf",
+                relative_path=generate_quality_report_filename(
+                    dataset_id=self.constants.dataset_id
+                ),
             )

dkist_processing_common/tasks/trial_output_data.py

@@ -100,7 +100,7 @@ class TransferTrialData(TransferDataBase, GlobusMixin):
         tag_list = []
         tag_list += [[Tag.output(), Tag.dataset_inventory()]]
         tag_list += [[Tag.output(), Tag.asdf()]]
-        tag_list += [[Tag.quality_data()]]  # quality data is not tagged as OUTPUT
+        tag_list += [[Tag.output(), Tag.quality_data()]]
         tag_list += [[Tag.output(), Tag.quality_report()]]
         tag_list += [[Tag.output(), Tag.movie()]]
         return tag_list

dkist_processing_common/tests/mock_metadata_store.py

@@ -6,6 +6,7 @@ import json
 from abc import ABC
 from abc import abstractmethod
 from datetime import datetime
+from datetime import timedelta
 from pathlib import Path
 from uuid import uuid4
 
@@ -134,10 +135,6 @@ class InputDatasetRecipeRunResponseMapping(ResponseMapping):
         return Unset
 
 
-class QualityResponseMapping(ResponseMapping):
-    pass  # TODO
-
-
 def make_default_recipe_run_status_response() -> RecipeRunStatusResponse:
     return RecipeRunStatusResponse(recipeRunStatusId=1)
 
@@ -234,3 +231,41 @@ def fake_gql_client():
     Convenience fixture for default mock GQL client. To customize, use fake_gql_client_factory.
     """
     return fake_gql_client_factory()
+
+
+def input_dataset_parameters_part_factory(
+    parameter_count: int = 1,
+    parameter_value_count: int = 1,
+    has_date: bool = False,
+    has_file: bool = False,
+) -> list[dict]:
+    """Create a mock InputDatasetPartDocumentList with parameters."""
+    result = [
+        {
+            "parameterName": uuid4().hex[:6],
+            "parameterValues": [
+                {"parameterValueId": i, "parameterValue": json.dumps(uuid4().hex)}
+                for i in range(parameter_value_count)
+            ],
+        }
+        for _ in range(parameter_count)
+    ]
+    if has_date:
+        base = datetime(2018, 9, 14, 0, 0, 0)  # This date is before any possible start dates
+        for parameter_index, data in enumerate(result):
+            for item in data["parameterValues"]:
+                dt = base + timedelta(days=parameter_index)
+                item["parameterValueStartDate"] = dt.isoformat()
+    if has_file:
+        for data in result:
+            param_list = data["parameterValues"]
+            for item in param_list:
+                item["parameterValue"] = json.dumps(
+                    {
+                        "__file__": {
+                            "bucket": "data",
+                            "objectKey": f"parameters/{data['parameterName']}/{uuid4().hex}.dat",
+                        }
+                    }
+                )
+    return result

dkist_processing_common/tests/test_input_dataset.py

@@ -1,5 +1,4 @@
 import json
-from datetime import datetime
 from typing import Any
 from uuid import uuid4
 
@@ -8,6 +7,7 @@ import pytest
 from dkist_processing_common.codecs.basemodel import basemodel_decoder
 from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.tests.mock_metadata_store import input_dataset_parameters_part_factory
 
 
 def input_dataset_frames_part_factory(bucket_count: int = 1) -> list[dict]:
@@ -25,42 +25,6 @@ def flatten_frame_parts(frame_parts: list[dict]) -> list[tuple[str, str]]:
     return result
 
 
-def input_dataset_parameters_part_factory(
-    parameter_count: int = 1,
-    parameter_value_count: int = 1,
-    has_date: bool = False,
-    has_file: bool = False,
-) -> list[dict]:
-    result = [
-        {
-            "parameterName": uuid4().hex[:6],
-            "parameterValues": [
-                {"parameterValueId": i, "parameterValue": json.dumps(uuid4().hex)}
-                for i in range(parameter_value_count)
-            ],
-        }
-        for _ in range(parameter_count)
-    ]
-    if has_date:
-        for data in result:
-            param_list = data["parameterValues"]
-            for item in param_list:
-                item["parameterValueStartDate"] = datetime(2022, 9, 14).isoformat()
-    if has_file:
-        for data in result:
-            param_list = data["parameterValues"]
-            for item in param_list:
-                item["parameterValue"] = json.dumps(
-                    {
-                        "__file__": {
-                            "bucket": "data",
-                            "objectKey": f"parameters/{data['parameterName']}/{uuid4().hex}.dat",
-                        }
-                    }
-                )
-    return result
-
-
 @pytest.mark.parametrize(
     "input_dataset_parts",
     [

dkist_processing_common/tests/test_publish_catalog_messages.py

@@ -57,24 +57,3 @@ def test_object_messages(publish_catalog_and_quality_messages_task):
         assert message.body.conversationId == str(task.recipe_run_id)
         assert message.body.objectType == object_type
         assert message.body.groupId == task.constants.dataset_id
-
-
-def test_quality_report_message(publish_catalog_and_quality_messages_task):
-    """
-    :Given: a PublishCatalogAndQualityMessages task
-    :When: creating quality report message
-    :Then: the attributes are correctly populated
-    """
-    # Given
-    task, proposal_id = publish_catalog_and_quality_messages_task
-    # When
-    message = task.quality_report_message
-    # Then
-    assert isinstance(message, CreateQualityReportMessage)
-    assert message.body.bucket == task.destination_bucket
-    # objectName exists and can be evaluated as a valid path
-    assert message.body.objectName
-    _ = Path(message.body.objectName)
-    assert message.body.datasetId == task.constants.dataset_id
-    assert message.body.conversationId == str(task.recipe_run_id)
-    assert message.body.incrementDatasetCatalogReceiptCount is True

dkist_processing_common/tests/test_quality_mixin.py

@@ -1214,12 +1214,20 @@ def wavecal_weights(wavecal_input_wavelength) -> np.ndarray:
 
 
 @pytest.fixture(scope="session")
-def wavecal_fit_result(wavecal_input_wavelength) -> FitResult:
+def wavecal_fit_result(wavecal_input_wavelength, wavecal_input_spectrum) -> FitResult:
     wavelength_params = WavelengthParameters(
         crpix=1, crval=10.0, dispersion=1, grating_constant=1, order=1, incident_light_angle=0
     )
-    minimizer_result = MinimizerResult(residual=np.random.random(wavecal_input_wavelength.size))
-    return FitResult(wavelength_parameters=wavelength_params, minimizer_result=minimizer_result)
+
+    residuals = np.random.random(wavecal_input_wavelength.size)
+    residuals[-1] = np.nan
+    minimizer_result = MinimizerResult(residual=residuals)
+    return FitResult(
+        wavelength_parameters=wavelength_params,
+        minimizer_result=minimizer_result,
+        input_wavelength_vector=wavecal_input_wavelength,
+        input_spectrum=wavecal_input_spectrum,
+    )
 
 
 @pytest.mark.parametrize(

dkist_processing_common/tests/test_submit_dataset_metadata.py

@@ -95,13 +95,10 @@ def test_submit_dataset_metadata(
     mocker.patch(
         "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
     )
-    # intercept these two GraphQLClient calls so they can be confirmed
+    # intercept this GraphQLClient call so it can be confirmed
     mocked_metadata_store_add_dataset_receipt_account = mocker.patch.object(
         metadata_store.MetadataStoreMixin, "metadata_store_add_dataset_receipt_account"
     )
-    mocked_metadata_store_add_quality_data = mocker.patch.object(
-        metadata_store.MetadataStoreMixin, "metadata_store_add_quality_data"
-    )
     task = submit_dataset_metadata_task
 
     # When
@@ -109,4 +106,3 @@ def test_submit_dataset_metadata(
 
     # Then
     mocked_metadata_store_add_dataset_receipt_account.assert_called_once()
-    mocked_metadata_store_add_quality_data.assert_called_once()

dkist_processing_common/tests/test_trial_catalog.py

@@ -12,14 +12,30 @@ from sqids import Sqids
 
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.asdf import asdf_decoder
+from dkist_processing_common.codecs.basemodel import basemodel_encoder
 from dkist_processing_common.codecs.bytes import bytes_decoder
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
 from dkist_processing_common.codecs.json import json_decoder
 from dkist_processing_common.codecs.quality import quality_data_encoder
+from dkist_processing_common.models.input_dataset import InputDatasetParameter
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks import CreateTrialAsdf
 from dkist_processing_common.tasks import CreateTrialDatasetInventory
 from dkist_processing_common.tasks import CreateTrialQualityReport
+from dkist_processing_common.tests.mock_metadata_store import input_dataset_parameters_part_factory
+
+
+@pytest.fixture()
+def mock_input_dataset_parts() -> InputDatasetPartDocumentList:
+    """An InputDatasetPartDocumentList with two parameters, each with one value and a date."""
+    raw = input_dataset_parameters_part_factory(
+        parameter_count=2,
+        parameter_value_count=1,
+        has_date=True,
+        has_file=False,
+    )
+    return InputDatasetPartDocumentList.model_validate({"doc_list": raw})
 
 
 @pytest.fixture()
@@ -41,6 +57,24 @@ def scratch_with_l1_frames(recipe_run_id, tmp_path) -> WorkflowFileSystem:
         scratch.write(
             file_obj, tags=[Tag.output(), Tag.frame()], relative_path=f"{uuid4().hex}.dat"
         )
+
+    return scratch
+
+
+@pytest.fixture()
+def scratch_with_l1_frames_and_parameters(
+    scratch_with_l1_frames, mock_input_dataset_parts
+) -> WorkflowFileSystem:
+    """Scratch instance for a recipe run id with tagged L1 frames and input parameters."""
+    scratch = scratch_with_l1_frames
+
+    # Write validated Pydantic model bytes expected by InputDatasetPartDocumentList
+    file_obj = basemodel_encoder(mock_input_dataset_parts)
+    scratch.write(
+        file_obj,
+        tags=Tag.input_dataset_parameters(),
+        relative_path=f"{uuid4().hex}.json",
+    )
     return scratch
 
 
@@ -85,6 +119,22 @@ def create_trial_asdf_task(
     task._purge()
 
 
+@pytest.fixture(scope="function")
+def create_trial_asdf_task_with_params(
+    recipe_run_id, tmp_path, scratch_with_l1_frames_and_parameters, fake_constants_db
+) -> CreateTrialAsdf:
+    """An instance of CreateTrialAsdf with L1 frames and input parameters tagged in scratch."""
+    task = CreateTrialAsdf(
+        recipe_run_id=recipe_run_id,
+        workflow_name="trial_asdf",
+        workflow_version="trial_asdf_version",
+    )
+    task.scratch = scratch_with_l1_frames_and_parameters
+    task.constants._update(fake_constants_db)
+    yield task
+    task._purge()
+
+
 @pytest.fixture()
 def create_trial_quality_report_task(
     recipe_run_id, tmp_path, fake_constants_db
@@ -143,25 +193,32 @@ def test_create_trial_dataset_inventory(create_trial_dataset_inventory_task):
     assert len(inventory) > 20  # a bunch
 
 
-def test_create_trial_asdf(create_trial_asdf_task, recipe_run_id):
+@pytest.mark.parametrize("with_params", [False, True], ids=["no_params", "with_params"])
+def test_create_trial_asdf(with_params, request, recipe_run_id, mock_input_dataset_parts):
     """
     :Given: An instance of CreateTrialAsdf with L1 frames tagged in scratch
     :When: CreateTrialAsdf is run
     :Then: An asdf file for the dataset is tagged in scratch
     """
-    task = create_trial_asdf_task
+    task = request.getfixturevalue(
+        "create_trial_asdf_task_with_params" if with_params else "create_trial_asdf_task"
+    )
     # When
     task()
+
     # Then
     asdf_tags = [Tag.output(), Tag.asdf()]
     filepaths = list(task.scratch.find_all(tags=asdf_tags))
     assert len(filepaths) == 1
     dataset_id = Sqids(min_length=6, alphabet=ascii_uppercase).encode([recipe_run_id])
     assert filepaths[0].name == f"INSTRUMENT_L1_20240416T160000_{dataset_id}_metadata.asdf"
+
     results = list(task.read(tags=asdf_tags, decoder=asdf_decoder))
     assert len(results) == 1
+
     tree = results[0]
     assert isinstance(tree, dict)
+
     for file_name in tree["dataset"].files.filenames:
         # This is a slightly better than check that `not Path(file_name).is_absolute()` because it confirms
         # we've correctly stripped the path of *all* parents (not just those that start at root).
@@ -169,6 +226,19 @@ def test_create_trial_asdf(create_trial_asdf_task, recipe_run_id):
         # `scratch.workflow_base_path`
         assert Path(file_name).name == file_name
 
+    # Only check parameters when present
+    ds = tree["dataset"]
+    assert "parameters" in ds.meta
+    parameters = ds.meta["parameters"]
+    assert isinstance(parameters, list)
+    if with_params:
+        assert parameters, f"ASDF tree must include input parameters: {parameters}"
+        assert len(parameters) == len(mock_input_dataset_parts.doc_list)
+        for param in parameters:
+            assert InputDatasetParameter.model_validate(param) in mock_input_dataset_parts.doc_list
+    else:
+        assert ds.meta["parameters"] == []
+
 
 def test_create_trial_quality_report(create_trial_quality_report_task):
     """

dkist_processing_common/tests/test_trial_output_data.py

@@ -158,13 +158,12 @@ def complete_trial_output_task(
         task.write(asdf_file_obj, relative_path=asdf_file_name, tags=[Tag.output(), Tag.asdf()])
 
         # Write quality data
-        # quality data is not tagged as OUTPUT
         quality_data_obj = uuid4().hex.encode("utf8")
         quality_data_name = "quality_data.json"
         task.write(
             quality_data_obj,
             relative_path=quality_data_name,
-            tags=Tag.quality_data(),
+            tags=[Tag.output(), Tag.quality_data()],
         )
 
         # Write a quality report file

{dkist_processing_common-11.7.0rc6.dist-info → dkist_processing_common-11.9.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 11.7.0rc6
+Version: 11.9.1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -17,12 +17,12 @@ Requires-Dist: asdf<4.0.0,>=3.5.0
 Requires-Dist: astropy>=7.0.0
 Requires-Dist: dkist-fits-specifications<5.0,>=4.0.0
 Requires-Dist: dkist-header-validator<6.0,>=5.0.0
-Requires-Dist: dkist-processing-core==6.0.0
+Requires-Dist: dkist-processing-core==6.0.1
 Requires-Dist: dkist-processing-pac<4.0,>=3.1
 Requires-Dist: dkist-service-configuration<5.0,>=4.1.7
 Requires-Dist: dkist-spectral-lines<4.0,>=3.0.0
-Requires-Dist: solar-wavelength-calibration<2.0,>=1.0
-Requires-Dist: globus-sdk<4.0.0,>=3.12.0
+Requires-Dist: solar-wavelength-calibration<3.0,>=2.0.0
+Requires-Dist: globus-sdk<5.0.0,>=4.0.0
 Requires-Dist: gqlclient[pydantic]==1.2.3
 Requires-Dist: sqids==0.5.1
 Requires-Dist: matplotlib>=3.4
@@ -31,7 +31,7 @@ Requires-Dist: numpy>=1.26.4
 Requires-Dist: object-clerk==1.0.0
 Requires-Dist: pandas>=1.4.2
 Requires-Dist: pillow>=10.2.0
-Requires-Dist: pydantic>=2.0
+Requires-Dist: pydantic>=2.7.2
 Requires-Dist: redis==6.4.0
 Requires-Dist: requests>=2.23
 Requires-Dist: scipy>=1.15.1
@@ -57,9 +57,9 @@ Requires-Dist: pytest; extra == "docs"
 Requires-Dist: towncrier<22.12.0; extra == "docs"
 Requires-Dist: dkist-sphinx-theme; extra == "docs"
 Provides-Extra: inventory
-Requires-Dist: dkist-inventory<2.0,>=1.6.0; extra == "inventory"
+Requires-Dist: dkist-inventory<2.0,>=1.11.2; extra == "inventory"
 Provides-Extra: asdf
-Requires-Dist: dkist-inventory[asdf]<2.0,>=1.6.0; extra == "asdf"
+Requires-Dist: dkist-inventory[asdf]<2.0,>=1.11.2; extra == "asdf"
 Provides-Extra: quality
 Requires-Dist: dkist-quality<3.0,>=2.0.0; extra == "quality"
 
@@ -118,6 +118,10 @@ Environment Variables
      - annotation=str required=False default='unknown-service-version' alias_priority=2 validation_alias='DKIST_SERVICE_VERSION' description='Service version for OpenTelemetry'
    * - NOMAD_ALLOC_ID
      - annotation=str required=False default='unknown-allocation-id' alias_priority=2 validation_alias='NOMAD_ALLOC_ID' description='Nomad allocation ID for OpenTelemetry'
+   * - NOMAD_ALLOC_NAME
+     - annotation=str required=False default='unknown-allocation-name' alias='NOMAD_ALLOC_NAME' alias_priority=2 description='Allocation name for the deployed container the task is running on.'
+   * - NOMAD_GROUP_NAME
+     - annotation=str required=False default='unknown-allocation-group' alias='NOMAD_GROUP_NAME' alias_priority=2 description='Allocation group for the deployed container the task is running on'
    * - OTEL_EXPORTER_OTLP_TRACES_INSECURE
      - annotation=bool required=False default=True description='Use insecure connection for OTLP traces'
    * - OTEL_EXPORTER_OTLP_METRICS_INSECURE
@@ -158,12 +162,12 @@ Environment Variables
      - annotation=Union[dict, NoneType] required=False default=None description='S3 upload configuration for the object store.'
    * - S3_DOWNLOAD_CONFIG
      - annotation=Union[dict, NoneType] required=False default=None description='S3 download configuration for the object store.'
-   * - GLOBUS_TRANSPORT_PARAMS
-     - annotation=dict required=False default_factory=dict description='Globus transfer parameters.'
-   * - GLOBUS_CLIENT_ID
-     - annotation=Union[str, NoneType] required=False default=None description='Globus client ID for inbound/outbound transfers.'
-   * - GLOBUS_CLIENT_SECRET
-     - annotation=Union[str, NoneType] required=False default=None description='Globus client secret for inbound/outbound transfers.'
+   * - GLOBUS_MAX_RETRIES
+     - annotation=int required=False default=5 description='Max retries for transient errors on calls to the globus api.'
+   * - GLOBUS_INBOUND_CLIENT_CREDENTIALS
+     - annotation=list[GlobusClientCredential] required=False default_factory=list description='Globus client credentials for inbound transfers.' examples=[[{'client_id': 'id1', 'client_secret': 'secret1'}, {'client_id': 'id2', 'client_secret': 'secret2'}]]
+   * - GLOBUS_OUTBOUND_CLIENT_CREDENTIALS
+     - annotation=list[GlobusClientCredential] required=False default_factory=list description='Globus client credentials for outbound transfers.' examples=[[{'client_id': 'id3', 'client_secret': 'secret3'}, {'client_id': 'id4', 'client_secret': 'secret4'}]]
    * - OBJECT_STORE_ENDPOINT
      - annotation=Union[str, NoneType] required=False default=None description='Object store Globus Endpoint ID.'
    * - SCRATCH_ENDPOINT

{dkist_processing_common-11.7.0rc6.dist-info → dkist_processing_common-11.9.1.dist-info}/RECORD

@@ -1,12 +1,6 @@
 changelog/.gitempty,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-changelog/267.feature.1.rst,sha256=G5Xl7EwiRKAtftSUvalfF7iUL-WgqIvnQ3k6wyTYzaw,98
-changelog/267.feature.2.rst,sha256=1z0TqX3159m-wFvEgElBjUp0XPWQvmqMlVlf4BzO-J8,99
-changelog/267.feature.rst,sha256=g4bBK3EFQiQljW9lQrFh6rVOL8C6yxBYbXI4WOH8NE0,65
-changelog/267.misc.rst,sha256=ciUqUesBbdxh41euyLhyD3Bl6IVT2TSIGeOg3LFjIvg,105
-changelog/267.removal.1.rst,sha256=qiGl1FAy9ioLwfCh8EvM788wjFbOY3jM3eUJHPOzpu8,207
-changelog/267.removal.rst,sha256=ecpctwJ-AE59sfbsTy-j3GYaPaN8h6oWphXUMNQ42Yw,103
 dkist_processing_common/__init__.py,sha256=GQ9EBnYhkOnt-qODclAoLS_g5YVhurxfg1tjVtI9rDI,320
-dkist_processing_common/config.py,sha256=kkFm-Q6jGGoV3Zdf3PSfk3xNYd2vSH0J2R8dBBav3Og,5218
+dkist_processing_common/config.py,sha256=f511KVpK24sQO4dDr4L6PMj5dz0jmWgnx2Y-3DpV0cw,5991
 dkist_processing_common/manual.py,sha256=bIVVyLsbXMh-g_2L3kGROL-1TtJe0_XviHsp7Br31x8,7023
 dkist_processing_common/_util/__init__.py,sha256=xf6JNpMKQgbhE2Jivymt-WO0WF6PpGt9rl604YpuTWk,92
 dkist_processing_common/_util/constants.py,sha256=0_bWLsvusHD8GrTx4B6V7AieKAaFbN9crcAALaS8x5Q,3245
@@ -32,7 +26,7 @@ dkist_processing_common/models/dkist_location.py,sha256=6Nk0wvv4R8ptlrV7BXon7abq
 dkist_processing_common/models/fits_access.py,sha256=imKqL4-_g6gTR-IeIjZ6qkMhQX3JujdrKFrTd9gOXnw,5605
 dkist_processing_common/models/flower_pot.py,sha256=_J7DwHM8u5kQfdPCpk5pUmALtLrM1L_h-x8JW5BSjXA,5129
 dkist_processing_common/models/fried_parameter.py,sha256=ro_H2Eo3I88lRf1wJjZfTc_XOjhgLt4whIQR_sjAFbM,1609
-dkist_processing_common/models/graphql.py,sha256=oSEcdVsVRytnIDEORLs4b6r3C2Lr7gqEEMHu82zpOOg,5912
+dkist_processing_common/models/graphql.py,sha256=QsKLbytpw_Qg9pJASscA7dZRfDbHLkpLZaWeqaHUDvo,5133
 dkist_processing_common/models/input_dataset.py,sha256=19w_ydrxdzjJgpnhFELqUomr7GixURjzLOaX41ipOKk,4173
 dkist_processing_common/models/message.py,sha256=ZEsPQalo5aKTOHfc5I15mNCe1KQcfJ3ivU7XBf8wnkM,1684
 dkist_processing_common/models/message_queue_binding.py,sha256=Y4otwkkePrLRSjlrya8nlEaBvCCUgfGZAWZF9XqCQ9Y,1012
@@ -64,26 +58,26 @@ dkist_processing_common/parsers/wavelength.py,sha256=P5C9mG8DAKK3GB3vWNRBI5l7pAW
 dkist_processing_common/tasks/__init__.py,sha256=l23ctjNsKJbHbbqaZBMeOPaOtw0hmITEljI_JJ-CVsU,627
 dkist_processing_common/tasks/assemble_movie.py,sha256=1ixDG-f4ODt0vywqVccG3aodLljVO5OGlvuMO9EEvcU,12767
 dkist_processing_common/tasks/base.py,sha256=itAHCvzcodo-q8_AjpWoRaM86BlcjWDpCIiUP7uwmP0,13236
-dkist_processing_common/tasks/l1_output_data.py,sha256=D4S3kH2uRKp1b8_xF2YiWO_mGj19UJS5wKQZ4OdatGs,10568
+dkist_processing_common/tasks/l1_output_data.py,sha256=eF3BvTTH0Bb163_gpJ8epxkjve8YIdrYgPZO_LCMKKo,10717
 dkist_processing_common/tasks/output_data_base.py,sha256=r1Bu3FX5zTVj66GTMWtaV_NdhxjyjSm661Bt2Mxmfi4,3685
 dkist_processing_common/tasks/parse_l0_input_data.py,sha256=KguXT0Xavynu7C8NFMjsV4628LRoTvfeSuApb6v4Neg,18835
 dkist_processing_common/tasks/quality_metrics.py,sha256=cvGF6tJ8yAvxOvkeG3tWxYwL885BrFW5X3V7_MSzL-A,12481
 dkist_processing_common/tasks/teardown.py,sha256=rwT9lWINVDF11-az_nx-Z5ykMTX_SJCchobpU6sErgk,2360
 dkist_processing_common/tasks/transfer_input_data.py,sha256=DAYfS-B1o-iBT9MXU-TiJG4Hv05Z0c_JzPrnFgvnK9g,5786
-dkist_processing_common/tasks/trial_catalog.py,sha256=iAaMT_oLnupA1O3xAtqVjsqRY5f_hyvMps-fXg6KlHU,8729
-dkist_processing_common/tasks/trial_output_data.py,sha256=CPMXXODvN5RTcu9bTF8v6AXciCl212EWP6qTiARvUNk,6837
+dkist_processing_common/tasks/trial_catalog.py,sha256=Yf-BKNCT_OHwJsxxZP8p2eRW04CcY0tw5_YIe1e9RQY,10535
+dkist_processing_common/tasks/trial_output_data.py,sha256=pUdrNlAzuir4AUdfax5_MOplB-A9NrXErMJmAwtJmLA,6811
 dkist_processing_common/tasks/write_l1.py,sha256=Xy834RTp3F95kLcW4ba5gfHMUocfZd82ZQQKnvQcP2M,23204
 dkist_processing_common/tasks/mixin/__init__.py,sha256=-g-DQbU7m1bclJYuFe3Yh757V-35GIDTbstardKQ7nU,68
-dkist_processing_common/tasks/mixin/globus.py,sha256=9ey_UCacqCfmxYZSgm6VDefdlm7dkNloC8G5DeVub8s,6592
+dkist_processing_common/tasks/mixin/globus.py,sha256=ugejtZ_MR5LesQYuXM1uICd_yWDE7cZZr0qnWCh75R8,6732
 dkist_processing_common/tasks/mixin/interservice_bus.py,sha256=M6R922l7gJSmmU_vswUXxy-c5DWNrIRjQu9H9CSgGfU,1081
-dkist_processing_common/tasks/mixin/metadata_store.py,sha256=QeZV3BITUzCErUt7mbkJME5SQWi3UPp_tw3zJ2PWy-M,11715
-dkist_processing_common/tasks/mixin/object_store.py,sha256=zGAyRR1O6EN52p7sFQtDWxa7ASx3HJSsO4d4tJr-beE,3237
+dkist_processing_common/tasks/mixin/metadata_store.py,sha256=GLfh0a0ehuRC8Uml59PfLExuoxGRj6tzNMytlO5jZf4,9106
+dkist_processing_common/tasks/mixin/object_store.py,sha256=l2sPzolmKsuYvUocHayT7PScliiFRIV12qIFYtCLII8,3888
 dkist_processing_common/tasks/mixin/quality/__init__.py,sha256=GOI_PBUxTmYp5IIuYFbwpA5Vx0jUwpdBBYYrnZMTh0E,384
 dkist_processing_common/tasks/mixin/quality/_base.py,sha256=qt9TZZ140skFWFmabrjlGdm60OLWEfx_xZAaohr6dLM,8492
-dkist_processing_common/tasks/mixin/quality/_metrics.py,sha256=k0hlthVbW7Jso9q_P0-hWGfp190kQO_oUQyGBRvxgqo,60626
+dkist_processing_common/tasks/mixin/quality/_metrics.py,sha256=TtM6V7qy0U6ofFmjXNoJKkaECoPSfJsU4Ziu1UQgPZs,60457
 dkist_processing_common/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dkist_processing_common/tests/conftest.py,sha256=Tm-Yq956EAafpDtu1d7JjdVY0Unp9e4z9ak-yf4wnH4,22320
-dkist_processing_common/tests/mock_metadata_store.py,sha256=fbCvSk1-s0ojN6l538RWodPW7dx6k4eXqipemnHKO0Y,8248
+dkist_processing_common/tests/mock_metadata_store.py,sha256=i41xu5AY4A566YT0qSzhP7qpR0wZp_EFH-OFcAsV4vQ,9591
 dkist_processing_common/tests/test_assemble_movie.py,sha256=dyVhowxB-Kc6GuxlDs74UrPtK9fwdUL7y5haA3Bidz0,4065
 dkist_processing_common/tests/test_assemble_quality.py,sha256=-F22jMY6mPy65VZ1TZY2r1vsxMXOPmZHArGx70OD3BA,17832
 dkist_processing_common/tests/test_base.py,sha256=gsyBG2R6Ufx7CzbHeGMagUwM9yCfpN4gCSZ6-aH2q48,6643
@@ -94,27 +88,27 @@ dkist_processing_common/tests/test_dkist_location.py,sha256=-_OoSw4SZDLFyIuOltHv
 dkist_processing_common/tests/test_fits_access.py,sha256=a50B4IAAH5NH5zeudTqyy0b5uWKJwJuzQLUdK1LoOHM,12832
 dkist_processing_common/tests/test_flower_pot.py,sha256=X9_UI3maa3ZQncV3jYHgovWnawDsdEkEB5vw6EAB96o,3151
 dkist_processing_common/tests/test_fried_parameter.py,sha256=iXtlQIifZ6cDOkEi-YDgP3oAlss2loq08Uohgvy1byQ,1295
-dkist_processing_common/tests/test_input_dataset.py,sha256=pQ01rWAkQ2XQojyHWzAqeOdrMXshNcgEVL5I_9bBTdo,9610
+dkist_processing_common/tests/test_input_dataset.py,sha256=wnQbZxBYywG5CEXces2WWk6I0QA7HjStaYSTVVbe5r0,8499
 dkist_processing_common/tests/test_interservice_bus.py,sha256=QrBeZ8dh497h6nxA8-aVUIGDcSj8y9DIXIk9I_HkXr0,3001
 dkist_processing_common/tests/test_interservice_bus_mixin.py,sha256=IptJkW7Qeu2Y742NKXEgkok2VdS600keLgCD3Y9iw3A,4131
 dkist_processing_common/tests/test_manual_processing.py,sha256=iHF7yQPlar9niYAGXtFv28Gw3Undlds38yMfszk4ccY,1037
 dkist_processing_common/tests/test_output_data_base.py,sha256=D8b1XKvbE3C5cGOiHq58yJ2pzQL3iL0wLZy_AkDdB9Y,3085
 dkist_processing_common/tests/test_parameters.py,sha256=CUEUIGBPMCUXPll0G0UxFDbMXi8lmnjRwXBarGX1PAQ,14033
 dkist_processing_common/tests/test_parse_l0_input_data.py,sha256=9OOqeMX8ReQO67ldoMHOBKLQg7Nd5qWHOEoHygcN5Ic,11889
-dkist_processing_common/tests/test_publish_catalog_messages.py,sha256=l6Wga1s2wNBIf4wGZ78ZIO_rtqjdidmtvlN9nMnQUAs,3222
+dkist_processing_common/tests/test_publish_catalog_messages.py,sha256=7WRsEwoLHGeaCmLTAW4tU_BlZw0e3hwx65uWSGzfuYE,2393
 dkist_processing_common/tests/test_quality.py,sha256=IPz7liXcmoqWIsY78oX07Ui0nWHxoUH2FbKGEmMle7E,10258
-dkist_processing_common/tests/test_quality_mixin.py,sha256=qanm3SXEiLb0OJDawbh8ixVAG9uRglFMzwxTeYxkDsM,55369
+dkist_processing_common/tests/test_quality_mixin.py,sha256=L-_kSIKs8A48LGt9QaItZWZqIcRF0MhBCAZQZYdSflk,55575
 dkist_processing_common/tests/test_scratch.py,sha256=WO8C1VJlkcC5IzST9Hj08CyyrINwYcN8pyteD9x38xs,16482
 dkist_processing_common/tests/test_stems.py,sha256=p__51u-b8vfWLI71aLxF3w2tcWtv6M6DyHJ7_6FZMHI,38949
-dkist_processing_common/tests/test_submit_dataset_metadata.py,sha256=LHEyjoIxJHXXssqKkr8Qn1NzzHD1FLJiD3lP8yaLiXU,3764
+dkist_processing_common/tests/test_submit_dataset_metadata.py,sha256=-UicRcyRQAC9H3sbTYlJaH4-Yn6jKNyQEZhzZxojzqw,3543
 dkist_processing_common/tests/test_tags.py,sha256=w5gmVfp3Ck92KNV80lJQRMz0OYgTYzWtwVUFWv1b5i8,5024
 dkist_processing_common/tests/test_task_name.py,sha256=kqFr59XX2K87xzfTlClzDV4-Je1dx72LvdaJ22UE8UU,1233
 dkist_processing_common/tests/test_task_parsing.py,sha256=2_OOmeZQWD17XAd_ECYmodJzD_iRIBKjCYdGh38BOx4,4421
 dkist_processing_common/tests/test_teardown.py,sha256=DaliHSGsiQBZaFkf5wb3XBo6rHNPmx2bmQtVymYeBN4,5601
 dkist_processing_common/tests/test_transfer_input_data.py,sha256=eyAAWXpTHQ8aew87-MncWpYBn4DAZrTSOL3LvlQfR5Q,12611
 dkist_processing_common/tests/test_transfer_l1_output_data.py,sha256=PVGDJBEUk4kAeu8ivrhlCE7yd29R18t9kZLFx-mpBwY,2063
-dkist_processing_common/tests/test_trial_catalog.py,sha256=Efq4yP1_0jccCFTi0_inyf4utC-oJVpGoAZ13fBP7I0,6778
-dkist_processing_common/tests/test_trial_output_data.py,sha256=YwmSfhNsSG-kdnvNlq7Ku0PNi_onTvU6uxn_nLiAKZw,12008
+dkist_processing_common/tests/test_trial_catalog.py,sha256=CxjtVABE5Fw2EvyXR56IJ3PPi9QvEOjccH0OzzRWk30,9424
+dkist_processing_common/tests/test_trial_output_data.py,sha256=fu3iGNV_FI8LOacezyt4HvXnxY3g1_UiBuRI63yz5Oo,11977
 dkist_processing_common/tests/test_workflow_task_base.py,sha256=LTVusltNrsGUOvw9G323am4CXebgE4tJhP6gZCcS0CQ,10457
 dkist_processing_common/tests/test_write_l1.py,sha256=alN-lozKEm6vKNdhtvzjnuPqv-NjHyUg16Op7SkMH-c,27964
 docs/Makefile,sha256=qnlVz6PuBqE39NfHWuUnHhNEA-EFgT2-WJNNNy9ttfk,4598
@@ -125,7 +119,7 @@ docs/landing_page.rst,sha256=aPAuXFhBx73lEZ59B6E6JXxkK0LlxzD0n-HXqHrfumQ,746
 docs/make.bat,sha256=mBAhtURwhQ7yc95pqwJzlhqBSvRknr1aqZ5s8NKvdKs,4513
 docs/requirements.txt,sha256=Kbl_X4c7RQZw035YTeNB63We6I7pvXFU4T0Uflp2yDY,29
 licenses/LICENSE.rst,sha256=piZaQplkzOMmH1NXg6QIdo9wwo9pPCoHkvm2-DmH76E,1462
-dkist_processing_common-11.7.0rc6.dist-info/METADATA,sha256=6JeOGbgzvzWIZH1VFNxUDvdoxhCDulcTuAjq8aEIebQ,13323
-dkist_processing_common-11.7.0rc6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dkist_processing_common-11.7.0rc6.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
-dkist_processing_common-11.7.0rc6.dist-info/RECORD,,
+dkist_processing_common-11.9.1.dist-info/METADATA,sha256=Ydji9eb7FYmBGirF5MWTn7tsBC58ULxT-9SOhF7Am5E,14062
+dkist_processing_common-11.9.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dkist_processing_common-11.9.1.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
+dkist_processing_common-11.9.1.dist-info/RECORD,,

changelog/267.feature.1.rst

@@ -1 +0,0 @@
-Add new bud types TaskAverageBud and TaskBeginDateBud, which is based on new TaskDatetimeBudBase.

changelog/267.feature.2.rst

@@ -1 +0,0 @@
-Add new bud type TaskContributingIdsBud, based on ContributingIdsBud, for for specific task types.

changelog/267.feature.rst

@@ -1 +0,0 @@
-Add new buds to parsing for what will become the dataset extras.

changelog/267.misc.rst

@@ -1 +0,0 @@
-Rename TimeFlowerBase and TaskTimeBudBase to RoundTimeFlowerBase and TaskRoundTimeBudBase, respectively.

changelog/267.removal.1.rst

@@ -1,2 +0,0 @@
-Remove the `_set_metadata_key_value` method from FitsAccessBase.  Instead of using `setattr`, attributes
-are assigned explicitly in access classes using `MetadataKey` members in place of header key strings.

changelog/267.removal.rst

@@ -1 +0,0 @@
-Remove IdBud, which is just a TaskUniqueBud with the task set to observe, and therefore is not needed.