dkist-processing-common 11.7.0rc6__py3-none-any.whl → 11.9.3__py3-none-any.whl

dkist_processing_common/config.py

@@ -3,12 +3,20 @@
 from dkist_processing_core.config import DKISTProcessingCoreConfiguration
 from dkist_service_configuration.settings import DEFAULT_MESH_SERVICE
 from dkist_service_configuration.settings import MeshService
+from pydantic import BaseModel
 from pydantic import Field
 from talus import ConnectionRetryerFactory
 from talus import ConsumerConnectionParameterFactory
 from talus import ProducerConnectionParameterFactory
 
 
+class GlobusClientCredential(BaseModel):
+    """Globus client credential."""
+
+    client_id: str = Field(..., description="Globus client ID for transfers.")
+    client_secret: str = Field(..., description="Globus client secret for transfers.")
+
+
 class DKISTProcessingCommonConfiguration(DKISTProcessingCoreConfiguration):
     """Common configurations."""
 
@@ -40,14 +48,28 @@ class DKISTProcessingCommonConfiguration(DKISTProcessingCoreConfiguration):
         default=None, description="S3 download configuration for the object store."
     )
     # globus
-    globus_transport_params: dict = Field(
-        default_factory=dict, description="Globus transfer parameters."
+    globus_max_retries: int = Field(
+        default=5, description="Max retries for transient errors on calls to the globus api."
     )
-    globus_client_id: str | None = Field(
-        default=None, description="Globus client ID for inbound/outbound transfers."
+    globus_inbound_client_credentials: list[GlobusClientCredential] = Field(
+        default_factory=list,
+        description="Globus client credentials for inbound transfers.",
+        examples=[
+            [
+                {"client_id": "id1", "client_secret": "secret1"},
+                {"client_id": "id2", "client_secret": "secret2"},
+            ],
+        ],
     )
-    globus_client_secret: str | None = Field(
-        default=None, description="Globus client secret for inbound/outbound transfers."
+    globus_outbound_client_credentials: list[GlobusClientCredential] = Field(
+        default_factory=list,
+        description="Globus client credentials for outbound transfers.",
+        examples=[
+            [
+                {"client_id": "id3", "client_secret": "secret3"},
+                {"client_id": "id4", "client_secret": "secret4"},
+            ],
+        ],
     )
     object_store_endpoint: str | None = Field(
         default=None, description="Object store Globus Endpoint ID."

dkist_processing_common/models/constants.py

@@ -53,6 +53,7 @@ class BudName(StrEnum):
     dark_date_end = "DARK_DATE_END"
     solar_gain_date_end = "SOLAR_GAIN_DATE_END"
     polcal_date_end = "POLCAL_DATE_END"
+    dark_num_raw_frames_per_fpa = "DARK_NUM_RAW_FRAMES_PER_FPA"
     solar_gain_num_raw_frames_per_fpa = "SOLAR_GAIN_NUM_RAW_FRAMES_PER_FPA"
     polcal_num_raw_frames_per_fpa = "POLCAL_NUM_RAW_FRAMES_PER_FPA"
     solar_gain_telescope_tracking_mode = "SOLAR_GAIN_TELESCOPE_TRACKING_MODE"
@@ -328,6 +329,13 @@ class ConstantsBase:
         """Return the date end constant for the polcal task."""
         return self._db_dict[BudName.polcal_date_end]
 
+    @property
+    def dark_num_raw_frames_per_fpa(self) -> dict[float, list]:
+        """Return the dictionary of exposure times to number of raw frames per fpa."""
+        raw_return = self._db_dict[BudName.dark_num_raw_frames_per_fpa]
+        # convert nested lists to dictionary
+        return {k: v for k, v in raw_return}
+
     @property
     def solar_gain_num_raw_frames_per_fpa(self) -> int:
         """Return the number of raw frames per fpa constant for the solar gain task."""

dkist_processing_common/models/graphql.py

@@ -176,36 +176,3 @@ class RecipeRunProvenanceMutation(GraphqlBaseModel):
     libraryVersions: str
     workflowVersion: str
     codeVersion: str | None = None
-
-
-class QualityCreation(GraphqlBaseModel):
-    """Quality data creation record."""
-
-    datasetId: str
-    metricCode: str
-    facet: str | None = None
-    name: str | None = None
-    description: str | None = None
-    statement: str | None = None
-    # JSON array
-    warnings: str | None = None
-    # JSON objects
-    plotData: str | None = None
-    multiPlotData: str | None = None
-    tableData: str | None = None
-    histogramData: str | None = None
-    modmatData: str | None = None
-    raincloudData: str | None = None
-    efficiencyData: str | None = None
-
-
-class QualitiesRequest(GraphqlBaseModel):
-    """Query parameters for quality data."""
-
-    datasetId: str
-
-
-class QualityResponse(GraphqlBaseModel):
-    """Query Response for quality data."""
-
-    qualityId: int

dkist_processing_common/parsers/lookup_bud.py

@@ -0,0 +1,129 @@
+"""Simple bud that is used to set a constant to a mapping dictionary."""
+
+from collections import defaultdict
+from enum import StrEnum
+from typing import Any
+from typing import Callable
+from typing import DefaultDict
+
+from dkist_processing_common.models.flower_pot import SpilledDirt
+from dkist_processing_common.models.flower_pot import Stem
+from dkist_processing_common.models.tags import EXP_TIME_ROUND_DIGITS
+from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
+from dkist_processing_common.parsers.task import passthrough_header_ip_task
+
+
+class TimeLookupBud(Stem):
+    """
+    Bud that reads two header keys from all files and creates a dictionary mapping a time KEY value to sets of a VALUE value.
+
+    Parameters
+    ----------
+    constant_name
+        The name for the constant to be defined
+
+    key_metadata_key
+        The time metadata key for the resulting dictionary key
+
+    value_metadata_key
+        The metadata key for the resulting dictionary value
+    """
+
+    def __init__(
+        self,
+        constant_name: str,
+        key_metadata_key: str | StrEnum,
+        value_metadata_key: str | StrEnum,
+    ):
+        super().__init__(stem_name=constant_name)
+
+        if isinstance(key_metadata_key, StrEnum):
+            key_metadata_key = key_metadata_key.name
+        self.key_metadata_key = key_metadata_key
+        if isinstance(value_metadata_key, StrEnum):
+            value_metadata_key = value_metadata_key.name
+        self.value_metadata_key = value_metadata_key
+
+        self.mapping: DefaultDict[float, set[Any]] = defaultdict(set)
+
+    def setter(self, fits_obj: L0FitsAccess):
+        """
+        Update the mapping dictionary.
+
+        Parameters
+        ----------
+        fits_obj
+            The input fits object
+        Returns
+        -------
+        Updates the dictionary and returns None
+        """
+        key = getattr(fits_obj, self.key_metadata_key)
+        rounded_key = round(key, EXP_TIME_ROUND_DIGITS)
+        value = getattr(fits_obj, self.value_metadata_key)
+        self.mapping[rounded_key].add(value)
+        return None
+
+    def getter(self, key):
+        """
+        Get the dictionary mapping created by the setter converted into hashable nested tuples.
+
+        Parameters
+        ----------
+        key
+            The input key
+        Returns
+        -------
+        The mapping dictionary converted into hashable nested tuples
+        """
+        hashable_mapping = tuple((k, tuple(v)) for k, v in self.mapping.items())
+        return hashable_mapping
+
+
+class TaskTimeLookupBud(TimeLookupBud):
+    """
+    Subclass of `TimeLookupBud` that only considers objects that have specific task types.
+
+    Parameters
+    ----------
+    constant_name
+        The name for the constant to be defined
+
+    key_metadata_key
+        The time metadata key for the resulting dictionary key
+
+    value_metadata_key
+        The metadata key for the resulting dictionary value
+
+    ip_task_types
+        Only consider objects whose parsed header IP task type matches a string in this list
+
+    task_type_parsing_function
+        The function used to convert a header into an IP task type
+    """
+
+    def __init__(
+        self,
+        constant_name: str,
+        key_metadata_key: str | StrEnum,
+        value_metadata_key: str | StrEnum,
+        ip_task_types: str | list[str],
+        task_type_parsing_function: Callable = passthrough_header_ip_task,
+    ):
+        super().__init__(
+            constant_name=constant_name,
+            key_metadata_key=key_metadata_key,
+            value_metadata_key=value_metadata_key,
+        )
+        if isinstance(ip_task_types, str):
+            ip_task_types = [ip_task_types]
+        self.ip_task_types = [task.casefold() for task in ip_task_types]
+        self.parsing_function = task_type_parsing_function
+
+    def setter(self, fits_obj: L0FitsAccess):
+        """Ingest an object only if its parsed IP task type matches what's desired."""
+        task = self.parsing_function(fits_obj)
+        if task.casefold() in self.ip_task_types:
+            return super().setter(fits_obj)
+
+        return SpilledDirt

dkist_processing_common/tasks/l1_output_data.py

@@ -38,8 +38,9 @@ class L1OutputDataBase(OutputDataBase, ABC):
 
     @property
     def dataset_has_quality_data(self) -> bool:
-        """Return True if quality data has been persisted to the metadata-store."""
-        return self.metadata_store_quality_data_exists(dataset_id=self.constants.dataset_id)
+        """Return True if the dataset has quality data."""
+        path_count = self.count(tags=[Tag.output(), Tag.quality_data()])
+        return path_count > 0
 
     def rollback(self):
         """Warn that the metadata-store and the interservice bus retain the effect of this tasks execution.  Rolling back this task may not be achievable without other action."""
@@ -58,6 +59,9 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
             # Movie needs to be transferred separately as the movie headers need to go with it
             self.transfer_movie()
 
+        with self.telemetry_span("Upload quality data"):
+            self.transfer_quality_data()
+
         with self.telemetry_span("Upload science frames"):
             self.transfer_output_frames()
 
@@ -79,19 +83,14 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
     def transfer_movie(self):
         """Transfer the movie to the object store."""
         paths = list(self.read(tags=[Tag.output(), Tag.movie()]))
-        if len(paths) == 0:
-            logger.warning(
-                f"No movies found to upload for dataset. recipe_run_id={self.recipe_run_id}"
-            )
-            return
-        movie = paths[0]
-        if count := len(paths) > 1:
-            # note: this needs to be an error or the dataset receipt accounting will have an
-            # expected count > the eventual actual
+
+        count = len(paths)
+        if count != 1:
             raise RuntimeError(
-                f"Multiple movies found to upload.  Uploading the first one. "
-                f"{count=}, {movie=}, recipe_run_id={self.recipe_run_id}"
+                f"Expected exactly one movie to upload, found {count}. "
+                f"recipe_run_id={self.recipe_run_id}"
             )
+        movie = paths[0]
         logger.info(f"Uploading Movie: recipe_run_id={self.recipe_run_id}, {movie=}")
         movie_object_key = self.format_object_key(movie)
         self.object_store_upload_movie(
@@ -101,6 +100,33 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
             content_type="video/mp4",
         )
 
+    def transfer_quality_data(self):
+        """Transfer quality data to the object store."""
+        paths = list(self.read(tags=[Tag.output(), Tag.quality_data()]))
+        if len(paths) == 0:
+            logger.info(
+                f"No quality data found to upload for dataset. recipe_run_id={self.recipe_run_id}"
+            )
+            return
+
+        if count := len(paths) > 1:
+            # dataset inventory does not support multiple quality data object keys
+            raise RuntimeError(
+                f"Found multiple quality data files to upload.  Not supported."
+                f"{count=}, recipe_run_id={self.recipe_run_id}"
+            )
+
+        with self.telemetry_span(f"Uploading the trial quality data"):
+            path = paths[0]
+            logger.info(f"Uploading quality data: recipe_run_id={self.recipe_run_id}, {path=}")
+            quality_data_object_key = self.format_object_key(path)
+            self.object_store_upload_quality_data(
+                quality_data=path,
+                bucket=self.destination_bucket,
+                object_key=quality_data_object_key,
+                content_type="application/json",
+            )
+
 
 class AssembleQualityData(L1OutputDataBase, QualityMixin):
     """
@@ -128,7 +154,7 @@ class AssembleQualityData(L1OutputDataBase, QualityMixin):
         ):
             self.write(
                 quality_data,
-                tags=Tag.quality_data(),
+                tags=[Tag.output(), Tag.quality_data()],
                 encoder=quality_data_encoder,
                 relative_path=f"{self.constants.dataset_id}_quality_data.json",
             )
@@ -136,31 +162,18 @@ class AssembleQualityData(L1OutputDataBase, QualityMixin):
 
 class SubmitDatasetMetadata(L1OutputDataBase):
     """
-    Add quality data and receipt account to the metadata store.
+    Add receipt account to the metadata store.
 
-    Add the quality data to the Quality database.
     Add a Dataset Receipt Account record to Processing Support for use by the Dataset Catalog Locker.
-    Adds the number of files created during the calibration processing to the Processing Support table
+    Adds the number of files to be created during the calibration processing to the Processing Support table
     for use by the Dataset Catalog Locker.
     """
 
     def run(self) -> None:
         """Run method for this task."""
-        with self.telemetry_span(f"Storing quality data to metadata store"):
-            # each quality_data file is a list - this will combine the elements of multiple lists into a single list
-            quality_data = list(
-                chain.from_iterable(
-                    self.read(tags=Tag.quality_data(), decoder=quality_data_decoder)
-                )
-            )
-            self.metadata_store_add_quality_data(
-                dataset_id=self.constants.dataset_id, quality_data=quality_data
-            )
         with self.telemetry_span("Count Expected Outputs"):
             dataset_id = self.constants.dataset_id
             expected_object_count = self.count(tags=Tag.output())
-            if quality_data:
-                expected_object_count += 1
         logger.info(
             f"Adding Dataset Receipt Account: "
             f"{dataset_id=}, {expected_object_count=}, recipe_run_id={self.recipe_run_id}"
@@ -230,24 +243,12 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         messages = [CatalogObjectMessage(body=body) for body in message_bodies]
         return messages
 
-    @property
-    def quality_report_message(self) -> CreateQualityReportMessage:
-        """Create the Quality Report Message."""
-        file_name = Path(f"{self.constants.dataset_id}_quality_report.pdf")
-        body = CreateQualityReportMessageBody(
-            bucket=self.destination_bucket,
-            objectName=self.format_object_key(file_name),
-            conversationId=str(self.recipe_run_id),
-            datasetId=self.constants.dataset_id,
-            incrementDatasetCatalogReceiptCount=True,
-        )
-        return CreateQualityReportMessage(body=body)
-
     def run(self) -> None:
         """Run method for this task."""
         with self.telemetry_span("Gather output data"):
             frames = self.read(tags=self.output_frame_tags)
             movies = self.read(tags=[Tag.output(), Tag.movie()])
+            quality_data = self.read(tags=[Tag.output(), Tag.quality_data()])
         with self.telemetry_span("Create message objects"):
             messages = []
             messages += self.frame_messages(paths=frames)
@@ -256,7 +257,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
             object_message_count = len(messages) - frame_message_count
             dataset_has_quality_data = self.dataset_has_quality_data
             if dataset_has_quality_data:
-                messages.append(self.quality_report_message)
+                messages += self.object_messages(paths=quality_data, object_type="QDATA")
         with self.telemetry_span(
             f"Publish messages: {frame_message_count = }, {object_message_count = }, {dataset_has_quality_data = }"
         ):

dkist_processing_common/tasks/mixin/globus.py

@@ -9,6 +9,8 @@ from globus_sdk import ConfidentialAppAuthClient
 from globus_sdk import GlobusError
 from globus_sdk import TransferClient
 from globus_sdk import TransferData
+from globus_sdk.scopes import TransferScopes
+from globus_sdk.transport import RetryConfig
 
 from dkist_processing_common.config import common_configurations
 
@@ -31,27 +33,32 @@ class GlobusTransferItem:
 class GlobusMixin:
     """Mixin to add methods to a Task to support globus transfers."""
 
-    @property
-    def globus_transfer_client(self) -> TransferClient:
-        """Get the globus transfer client, creating it if it doesn't exist."""
-        if getattr(self, "_globus_transfer_client", False):
-            return self._globus_transfer_client
+    def globus_transfer_client_factory(self, transfer_data: TransferData) -> TransferClient:
+        """Create a globus transfer client based on the direction of transfer and round-robin the available application credentials."""
+        if (
+            transfer_data["source_endpoint"] == common_configurations.object_store_endpoint
+        ):  # inbound
+            client_credentials = common_configurations.globus_inbound_client_credentials
+        else:  # outbound
+            client_credentials = common_configurations.globus_outbound_client_credentials
+
+        # Round-robin the client credentials based on the recipe run id
+        index = self.recipe_run_id % len(client_credentials)
+        selected_credential = client_credentials[index]
+
         confidential_client = ConfidentialAppAuthClient(
-            client_id=common_configurations.globus_client_id,
-            client_secret=common_configurations.globus_client_secret,
-            transport_params=common_configurations.globus_transport_params,
+            client_id=selected_credential.client_id,
+            client_secret=selected_credential.client_secret,
         )
-        authorizer = ClientCredentialsAuthorizer(
-            confidential_client, scopes="urn:globus:auth:scope:transfer.api.globus.org:all"
-        )
-        self._globus_transfer_client = TransferClient(authorizer=authorizer)
-        return self._globus_transfer_client
+        authorizer = ClientCredentialsAuthorizer(confidential_client, scopes=TransferScopes)
+        retry_config = RetryConfig(max_retries=common_configurations.globus_max_retries)
+
+        return TransferClient(authorizer=authorizer, retry_config=retry_config)
 
     def globus_transfer_scratch_to_object_store(
         self,
         transfer_items: list[GlobusTransferItem],
         label: str = None,
-        sync_level: str = None,
         verify_checksum: bool = True,
     ) -> None:
         """Transfer data from scratch to the object store."""
@@ -60,7 +67,6 @@ class GlobusMixin:
             destination_endpoint=common_configurations.object_store_endpoint,
             transfer_items=transfer_items,
             label=label,
-            sync_level=sync_level,
             verify_checksum=verify_checksum,
         )
 
@@ -68,7 +74,6 @@ class GlobusMixin:
         self,
         transfer_items: list[GlobusTransferItem],
         label: str = None,
-        sync_level: str = None,
         verify_checksum: bool = True,
     ) -> None:
         """Transfer data from the object store to scratch."""
@@ -77,7 +82,6 @@ class GlobusMixin:
             destination_endpoint=common_configurations.scratch_endpoint,
             transfer_items=transfer_items,
             label=label,
-            sync_level=sync_level,
             verify_checksum=verify_checksum,
         )
 
@@ -87,7 +91,6 @@ class GlobusMixin:
         destination_endpoint: str,
         transfer_items: list[GlobusTransferItem],
         label: str = None,
-        sync_level: str = None,
         verify_checksum: bool = True,
     ) -> TransferData:
         """Format a globus TransferData instance."""
@@ -95,7 +98,6 @@ class GlobusMixin:
             source_endpoint=source_endpoint,
             destination_endpoint=destination_endpoint,
             label=label,
-            sync_level=sync_level,
             verify_checksum=verify_checksum,
         )
         for item in transfer_items:
@@ -112,7 +114,6 @@ class GlobusMixin:
         destination_endpoint: str,
         transfer_items: list[GlobusTransferItem],
         label: str = None,
-        sync_level: str = None,
         verify_checksum: bool = True,
     ) -> None:
         """Perform a transfer of data using globus."""
@@ -121,7 +122,6 @@ class GlobusMixin:
             destination_endpoint=destination_endpoint,
             transfer_items=transfer_items,
             label=label,
-            sync_level=sync_level,
             verify_checksum=verify_checksum,
         )
         self._blocking_globus_transfer(transfer_data=transfer_data)
@@ -131,24 +131,21 @@ class GlobusMixin:
         source_endpoint: str,
         destination_endpoint: str,
         label: str = None,
-        sync_level: str = None,
         verify_checksum: bool = True,
     ) -> TransferData:
         label = label or "Data Processing Transfer"
         return TransferData(
-            transfer_client=self.globus_transfer_client,
             source_endpoint=source_endpoint,
             destination_endpoint=destination_endpoint,
             label=label,
-            sync_level=sync_level,
             verify_checksum=verify_checksum,
         )
 
     def _blocking_globus_transfer(self, transfer_data: TransferData) -> None:
-        tc = self.globus_transfer_client
-        logger.info(f"Starting globus transfer: label={transfer_data.get('label')}")
+        tc = self.globus_transfer_client_factory(transfer_data=transfer_data)
         transfer_result = tc.submit_transfer(transfer_data)
         task_id = transfer_result["task_id"]
+        logger.info(f"Starting globus transfer: label={transfer_data.get('label')}, {task_id=}, ")
         polling_interval = 60
         while not tc.task_wait(
             task_id=task_id, timeout=polling_interval, polling_interval=polling_interval

dkist_processing_common/tasks/mixin/metadata_store.py

@@ -1,6 +1,5 @@
 """Mixin for a WorkflowDataTaskBase subclass which implements Metadata Store data access functionality."""
 
-import json
 import logging
 from functools import cached_property
 from typing import Literal
@@ -8,15 +7,11 @@ from typing import Literal
 from pydantic import validate_call
 
 from dkist_processing_common._util.graphql import GraphQLClient
-from dkist_processing_common.codecs.quality import QualityDataEncoder
 from dkist_processing_common.config import common_configurations
 from dkist_processing_common.models.graphql import DatasetCatalogReceiptAccountMutation
 from dkist_processing_common.models.graphql import DatasetCatalogReceiptAccountResponse
 from dkist_processing_common.models.graphql import InputDatasetPartResponse
 from dkist_processing_common.models.graphql import InputDatasetRecipeRunResponse
-from dkist_processing_common.models.graphql import QualitiesRequest
-from dkist_processing_common.models.graphql import QualityCreation
-from dkist_processing_common.models.graphql import QualityResponse
 from dkist_processing_common.models.graphql import RecipeRunMutation
 from dkist_processing_common.models.graphql import RecipeRunMutationResponse
 from dkist_processing_common.models.graphql import RecipeRunProvenanceMutation
@@ -150,50 +145,6 @@ class MetadataStoreMixin:
             mutation_response_cls=RecipeRunProvenanceResponse,
         )
 
-    # QUALITY
-
-    def metadata_store_add_quality_data(self, dataset_id: str, quality_data: list[dict]):
-        """Add the quality data to the metadata-store."""
-        if self.metadata_store_quality_data_exists(dataset_id):
-            raise RuntimeError(f"Quality data already persisted for dataset {dataset_id!r}")
-        for metric in quality_data:
-            if (metric_code := metric.get("metric_code")) is None:
-                name = metric.get("name")
-                raise ValueError(f"No metric_code for {name!r} in dataset {dataset_id!r}")
-            params = QualityCreation(
-                datasetId=dataset_id,
-                metricCode=metric_code,
-                facet=metric.get("facet"),
-                name=metric.get("name"),
-                description=metric.get("description"),
-                statement=metric.get("statement"),
-                # JSON array
-                warnings=json.dumps(metric.get("warnings")),
-                # JSON objects
-                plotData=json.dumps(metric.get("plot_data"), cls=QualityDataEncoder),
-                multiPlotData=json.dumps(metric.get("multi_plot_data"), cls=QualityDataEncoder),
-                tableData=json.dumps(metric.get("table_data"), cls=QualityDataEncoder),
-                histogramData=json.dumps(metric.get("histogram_data"), cls=QualityDataEncoder),
-                modmatData=json.dumps(metric.get("modmat_data"), cls=QualityDataEncoder),
-                raincloudData=json.dumps(metric.get("raincloud_data"), cls=QualityDataEncoder),
-                efficiencyData=json.dumps(metric.get("efficiency_data"), cls=QualityDataEncoder),
-            )
-            self.metadata_store_client.execute_gql_mutation(
-                mutation_base="createQuality",
-                mutation_parameters=params,
-                mutation_response_cls=QualityResponse,
-            )
-
-    def metadata_store_quality_data_exists(self, dataset_id: str) -> bool:
-        """Return True if quality data exists in the metadata-store for the given dataset id."""
-        params = QualitiesRequest(datasetId=dataset_id)
-        response = self.metadata_store_client.execute_gql_query(
-            query_base="qualities",
-            query_response_cls=QualityResponse,
-            query_parameters=params,
-        )
-        return bool(response)
-
     # INPUT DATASET RECIPE RUN
 
     @cached_property

dkist_processing_common/tasks/mixin/object_store.py

@@ -55,6 +55,27 @@ class ObjectStoreMixin:
             },
         )
 
+    def object_store_upload_quality_data(
+        self,
+        quality_data: Path | bytes,
+        bucket: str,
+        object_key: str,
+        content_type: str = "application/json",
+    ):
+        """Upload quality data to the object store."""
+        self.object_store_client.upload_object(
+            object_data=quality_data,
+            bucket=bucket,
+            object_key=object_key,
+            verify_checksum=True,
+            content_type=content_type,
+            metadata={
+                "groupname": "DATASET",
+                "groupid": self.constants.dataset_id,
+                "objecttype": "QDATA",
+            },
+        )
+
     def object_store_remove_folder_objects(self, bucket: str, path: Path | str) -> list[str]:
         """
         Remove folder objects (end with /) in the specified bucket and path.

dkist_processing_common/tasks/mixin/quality/_metrics.py

@@ -1356,15 +1356,13 @@ class _WavecalQualityMixin:
         Note that the residuals are the *unweighed* residuals.
         """
         weight_data = np.ones(input_wavelength.size) if weights is None else weights
-        prepared_weights = np.sqrt(weight_data / np.sum(weight_data))
+        prepared_weights = fit_result.prepared_weights
         residuals = fit_result.minimizer_result.residual / prepared_weights
         residuals[~np.isfinite(residuals)] = 0.0
-        best_fit_atlas = input_spectrum - residuals
         normalized_residuals = residuals / input_spectrum
 
-        best_fit_header = fit_result.wavelength_parameters.to_header(axis_num=1)
-        wcs = WCS(best_fit_header)
-        best_fit_wavelength = wcs.spectral.pixel_to_world(np.arange(input_spectrum.size))
+        best_fit_atlas = fit_result.best_fit_atlas
+        best_fit_wavelength = fit_result.best_fit_wavelength_vector
 
         finite_idx = (
             np.isfinite(input_wavelength)
@@ -1378,7 +1376,7 @@ class _WavecalQualityMixin:
         data = {
             "input_wavelength_nm": input_wavelength.to_value(u.nm)[finite_idx].tolist(),
             "input_spectrum": input_spectrum[finite_idx].tolist(),
-            "best_fit_wavelength_nm": best_fit_wavelength.to_value(u.nm)[finite_idx].tolist(),
+            "best_fit_wavelength_nm": best_fit_wavelength[finite_idx].tolist(),
             "best_fit_atlas": best_fit_atlas[finite_idx].tolist(),
             "normalized_residuals": normalized_residuals[finite_idx].tolist(),
             "weights": None if weights is None else weight_data[finite_idx].tolist(),