dkist-processing-common 11.9.0rc1__tar.gz → 11.9.1rc1__tar.gz

{dkist_processing_common-11.9.0rc1 → dkist_processing_common-11.9.1rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 11.9.0rc1
+Version: 11.9.1rc1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -57,9 +57,9 @@ Requires-Dist: pytest; extra == "docs"
 Requires-Dist: towncrier<22.12.0; extra == "docs"
 Requires-Dist: dkist-sphinx-theme; extra == "docs"
 Provides-Extra: inventory
-Requires-Dist: dkist-inventory<2.0,>=1.11.1; extra == "inventory"
+Requires-Dist: dkist-inventory==1.11.2; extra == "inventory"
 Provides-Extra: asdf
-Requires-Dist: dkist-inventory[asdf]<2.0,>=1.11.1; extra == "asdf"
+Requires-Dist: dkist-inventory[asdf]==1.11.2; extra == "asdf"
 Provides-Extra: quality
 Requires-Dist: dkist-quality<3.0,>=2.0.0; extra == "quality"
 

dkist_processing_common-11.9.1rc1/changelog/276.feature.rst

@@ -0,0 +1 @@
+Store quality data in object store.

{dkist_processing_common-11.9.0rc1 → dkist_processing_common-11.9.1rc1}/dkist_processing_common/models/graphql.py

@@ -176,36 +176,3 @@ class RecipeRunProvenanceMutation(GraphqlBaseModel):
     libraryVersions: str
     workflowVersion: str
     codeVersion: str | None = None
-
-
-class QualityCreation(GraphqlBaseModel):
-    """Quality data creation record."""
-
-    datasetId: str
-    metricCode: str
-    facet: str | None = None
-    name: str | None = None
-    description: str | None = None
-    statement: str | None = None
-    # JSON array
-    warnings: str | None = None
-    # JSON objects
-    plotData: str | None = None
-    multiPlotData: str | None = None
-    tableData: str | None = None
-    histogramData: str | None = None
-    modmatData: str | None = None
-    raincloudData: str | None = None
-    efficiencyData: str | None = None
-
-
-class QualitiesRequest(GraphqlBaseModel):
-    """Query parameters for quality data."""
-
-    datasetId: str
-
-
-class QualityResponse(GraphqlBaseModel):
-    """Query Response for quality data."""
-
-    qualityId: int

{dkist_processing_common-11.9.0rc1 → dkist_processing_common-11.9.1rc1}/dkist_processing_common/tasks/l1_output_data.py

@@ -38,8 +38,9 @@ class L1OutputDataBase(OutputDataBase, ABC):
 
     @property
     def dataset_has_quality_data(self) -> bool:
-        """Return True if quality data has been persisted to the metadata-store."""
-        return self.metadata_store_quality_data_exists(dataset_id=self.constants.dataset_id)
+        """Return True if the dataset has quality data."""
+        path_count = self.count(tags=[Tag.output(), Tag.quality_data()])
+        return path_count > 0
 
     def rollback(self):
         """Warn that the metadata-store and the interservice bus retain the effect of this tasks execution.  Rolling back this task may not be achievable without other action."""
@@ -58,6 +59,9 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
             # Movie needs to be transferred separately as the movie headers need to go with it
             self.transfer_movie()
 
+        with self.telemetry_span("Upload quality data"):
+            self.transfer_quality_data()
+
         with self.telemetry_span("Upload science frames"):
             self.transfer_output_frames()
 
@@ -101,6 +105,33 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
             content_type="video/mp4",
         )
 
+    def transfer_quality_data(self):
+        """Transfer quality data to the object store."""
+        paths = list(self.read(tags=[Tag.output(), Tag.quality_data()]))
+        if len(paths) == 0:
+            logger.info(
+                f"No quality data found to upload for dataset. recipe_run_id={self.recipe_run_id}"
+            )
+            return
+
+        if count := len(paths) > 1:
+            # dataset inventory does not support multiple quality data object keys
+            raise RuntimeError(
+                f"Found multiple quality data files to upload.  Not supported."
+                f"{count=}, recipe_run_id={self.recipe_run_id}"
+            )
+
+        with self.telemetry_span(f"Uploading the trial quality data"):
+            path = paths[0]
+            logger.info(f"Uploading quality data: recipe_run_id={self.recipe_run_id}, {path=}")
+            quality_data_object_key = self.format_object_key(path)
+            self.object_store_upload_quality_data(
+                quality_data=path,
+                bucket=self.destination_bucket,
+                object_key=quality_data_object_key,
+                content_type="application/json",
+            )
+
 
 class AssembleQualityData(L1OutputDataBase, QualityMixin):
     """
@@ -128,7 +159,7 @@ class AssembleQualityData(L1OutputDataBase, QualityMixin):
         ):
             self.write(
                 quality_data,
-                tags=Tag.quality_data(),
+                tags=[Tag.output(), Tag.quality_data()],
                 encoder=quality_data_encoder,
                 relative_path=f"{self.constants.dataset_id}_quality_data.json",
             )
@@ -136,31 +167,18 @@ class AssembleQualityData(L1OutputDataBase, QualityMixin):
 
 class SubmitDatasetMetadata(L1OutputDataBase):
     """
-    Add quality data and receipt account to the metadata store.
+    Add receipt account to the metadata store.
 
-    Add the quality data to the Quality database.
     Add a Dataset Receipt Account record to Processing Support for use by the Dataset Catalog Locker.
-    Adds the number of files created during the calibration processing to the Processing Support table
+    Adds the number of files to be created during the calibration processing to the Processing Support table
     for use by the Dataset Catalog Locker.
     """
 
     def run(self) -> None:
         """Run method for this task."""
-        with self.telemetry_span(f"Storing quality data to metadata store"):
-            # each quality_data file is a list - this will combine the elements of multiple lists into a single list
-            quality_data = list(
-                chain.from_iterable(
-                    self.read(tags=Tag.quality_data(), decoder=quality_data_decoder)
-                )
-            )
-            self.metadata_store_add_quality_data(
-                dataset_id=self.constants.dataset_id, quality_data=quality_data
-            )
         with self.telemetry_span("Count Expected Outputs"):
             dataset_id = self.constants.dataset_id
             expected_object_count = self.count(tags=Tag.output())
-            if quality_data:
-                expected_object_count += 1
         logger.info(
             f"Adding Dataset Receipt Account: "
             f"{dataset_id=}, {expected_object_count=}, recipe_run_id={self.recipe_run_id}"
@@ -230,24 +248,12 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         messages = [CatalogObjectMessage(body=body) for body in message_bodies]
         return messages
 
-    @property
-    def quality_report_message(self) -> CreateQualityReportMessage:
-        """Create the Quality Report Message."""
-        file_name = Path(f"{self.constants.dataset_id}_quality_report.pdf")
-        body = CreateQualityReportMessageBody(
-            bucket=self.destination_bucket,
-            objectName=self.format_object_key(file_name),
-            conversationId=str(self.recipe_run_id),
-            datasetId=self.constants.dataset_id,
-            incrementDatasetCatalogReceiptCount=True,
-        )
-        return CreateQualityReportMessage(body=body)
-
     def run(self) -> None:
         """Run method for this task."""
         with self.telemetry_span("Gather output data"):
             frames = self.read(tags=self.output_frame_tags)
             movies = self.read(tags=[Tag.output(), Tag.movie()])
+            quality_data = self.read(tags=[Tag.output(), Tag.quality_data()])
         with self.telemetry_span("Create message objects"):
             messages = []
             messages += self.frame_messages(paths=frames)
@@ -256,7 +262,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
             object_message_count = len(messages) - frame_message_count
             dataset_has_quality_data = self.dataset_has_quality_data
             if dataset_has_quality_data:
-                messages.append(self.quality_report_message)
+                messages += self.object_messages(paths=quality_data, object_type="QDATA")
         with self.telemetry_span(
             f"Publish messages: {frame_message_count = }, {object_message_count = }, {dataset_has_quality_data = }"
         ):

{dkist_processing_common-11.9.0rc1 → dkist_processing_common-11.9.1rc1}/dkist_processing_common/tasks/mixin/metadata_store.py

@@ -1,6 +1,5 @@
 """Mixin for a WorkflowDataTaskBase subclass which implements Metadata Store data access functionality."""
 
-import json
 import logging
 from functools import cached_property
 from typing import Literal
@@ -8,15 +7,11 @@ from typing import Literal
 from pydantic import validate_call
 
 from dkist_processing_common._util.graphql import GraphQLClient
-from dkist_processing_common.codecs.quality import QualityDataEncoder
 from dkist_processing_common.config import common_configurations
 from dkist_processing_common.models.graphql import DatasetCatalogReceiptAccountMutation
 from dkist_processing_common.models.graphql import DatasetCatalogReceiptAccountResponse
 from dkist_processing_common.models.graphql import InputDatasetPartResponse
 from dkist_processing_common.models.graphql import InputDatasetRecipeRunResponse
-from dkist_processing_common.models.graphql import QualitiesRequest
-from dkist_processing_common.models.graphql import QualityCreation
-from dkist_processing_common.models.graphql import QualityResponse
 from dkist_processing_common.models.graphql import RecipeRunMutation
 from dkist_processing_common.models.graphql import RecipeRunMutationResponse
 from dkist_processing_common.models.graphql import RecipeRunProvenanceMutation
@@ -150,50 +145,6 @@ class MetadataStoreMixin:
             mutation_response_cls=RecipeRunProvenanceResponse,
         )
 
-    # QUALITY
-
-    def metadata_store_add_quality_data(self, dataset_id: str, quality_data: list[dict]):
-        """Add the quality data to the metadata-store."""
-        if self.metadata_store_quality_data_exists(dataset_id):
-            raise RuntimeError(f"Quality data already persisted for dataset {dataset_id!r}")
-        for metric in quality_data:
-            if (metric_code := metric.get("metric_code")) is None:
-                name = metric.get("name")
-                raise ValueError(f"No metric_code for {name!r} in dataset {dataset_id!r}")
-            params = QualityCreation(
-                datasetId=dataset_id,
-                metricCode=metric_code,
-                facet=metric.get("facet"),
-                name=metric.get("name"),
-                description=metric.get("description"),
-                statement=metric.get("statement"),
-                # JSON array
-                warnings=json.dumps(metric.get("warnings")),
-                # JSON objects
-                plotData=json.dumps(metric.get("plot_data"), cls=QualityDataEncoder),
-                multiPlotData=json.dumps(metric.get("multi_plot_data"), cls=QualityDataEncoder),
-                tableData=json.dumps(metric.get("table_data"), cls=QualityDataEncoder),
-                histogramData=json.dumps(metric.get("histogram_data"), cls=QualityDataEncoder),
-                modmatData=json.dumps(metric.get("modmat_data"), cls=QualityDataEncoder),
-                raincloudData=json.dumps(metric.get("raincloud_data"), cls=QualityDataEncoder),
-                efficiencyData=json.dumps(metric.get("efficiency_data"), cls=QualityDataEncoder),
-            )
-            self.metadata_store_client.execute_gql_mutation(
-                mutation_base="createQuality",
-                mutation_parameters=params,
-                mutation_response_cls=QualityResponse,
-            )
-
-    def metadata_store_quality_data_exists(self, dataset_id: str) -> bool:
-        """Return True if quality data exists in the metadata-store for the given dataset id."""
-        params = QualitiesRequest(datasetId=dataset_id)
-        response = self.metadata_store_client.execute_gql_query(
-            query_base="qualities",
-            query_response_cls=QualityResponse,
-            query_parameters=params,
-        )
-        return bool(response)
-
     # INPUT DATASET RECIPE RUN
 
     @cached_property

{dkist_processing_common-11.9.0rc1 → dkist_processing_common-11.9.1rc1}/dkist_processing_common/tasks/mixin/object_store.py

@@ -55,6 +55,27 @@ class ObjectStoreMixin:
             },
         )
 
+    def object_store_upload_quality_data(
+        self,
+        quality_data: Path | bytes,
+        bucket: str,
+        object_key: str,
+        content_type: str = "application/json",
+    ):
+        """Upload quality data to the object store."""
+        self.object_store_client.upload_object(
+            object_data=quality_data,
+            bucket=bucket,
+            object_key=object_key,
+            verify_checksum=True,
+            content_type=content_type,
+            metadata={
+                "groupname": "DATASET",
+                "groupid": self.constants.dataset_id,
+                "objecttype": "QDATA",
+            },
+        )
+
     def object_store_remove_folder_objects(self, bucket: str, path: Path | str) -> list[str]:
         """
         Remove folder objects (end with /) in the specified bucket and path.

{dkist_processing_common-11.9.0rc1 → dkist_processing_common-11.9.1rc1}/dkist_processing_common/tasks/trial_catalog.py

@@ -30,6 +30,7 @@ INVENTORY_EXTRA_INSTALLED = False
 try:
     from dkist_inventory.inventory import generate_asdf_filename
     from dkist_inventory.inventory import generate_inventory_from_frame_inventory
+    from dkist_inventory.inventory import generate_quality_report_filename
 
     INVENTORY_EXTRA_INSTALLED = True
 except ModuleNotFoundError:
@@ -245,5 +246,7 @@ class CreateTrialQualityReport(OutputDataBase):
             self.write(
                 quality_report,
                 tags=[Tag.output(), Tag.quality_report()],
-                relative_path=f"{self.constants.dataset_id}_quality_report.pdf",
+                relative_path=generate_quality_report_filename(
+                    dataset_id=self.constants.dataset_id
+                ),
             )

{dkist_processing_common-11.9.0rc1 → dkist_processing_common-11.9.1rc1}/dkist_processing_common/tasks/trial_output_data.py

@@ -100,7 +100,7 @@ class TransferTrialData(TransferDataBase, GlobusMixin):
         tag_list = []
         tag_list += [[Tag.output(), Tag.dataset_inventory()]]
         tag_list += [[Tag.output(), Tag.asdf()]]
-        tag_list += [[Tag.quality_data()]]  # quality data is not tagged as OUTPUT
+        tag_list += [[Tag.output(), Tag.quality_data()]]
         tag_list += [[Tag.output(), Tag.quality_report()]]
         tag_list += [[Tag.output(), Tag.movie()]]
         return tag_list

{dkist_processing_common-11.9.0rc1 → dkist_processing_common-11.9.1rc1}/dkist_processing_common/tests/mock_metadata_store.py

@@ -135,10 +135,6 @@ class InputDatasetRecipeRunResponseMapping(ResponseMapping):
         return Unset
 
 
-class QualityResponseMapping(ResponseMapping):
-    pass  # TODO
-
-
 def make_default_recipe_run_status_response() -> RecipeRunStatusResponse:
     return RecipeRunStatusResponse(recipeRunStatusId=1)
 

{dkist_processing_common-11.9.0rc1 → dkist_processing_common-11.9.1rc1}/dkist_processing_common/tests/test_publish_catalog_messages.py

@@ -57,24 +57,3 @@ def test_object_messages(publish_catalog_and_quality_messages_task):
         assert message.body.conversationId == str(task.recipe_run_id)
         assert message.body.objectType == object_type
         assert message.body.groupId == task.constants.dataset_id
-
-
-def test_quality_report_message(publish_catalog_and_quality_messages_task):
-    """
-    :Given: a PublishCatalogAndQualityMessages task
-    :When: creating quality report message
-    :Then: the attributes are correctly populated
-    """
-    # Given
-    task, proposal_id = publish_catalog_and_quality_messages_task
-    # When
-    message = task.quality_report_message
-    # Then
-    assert isinstance(message, CreateQualityReportMessage)
-    assert message.body.bucket == task.destination_bucket
-    # objectName exists and can be evaluated as a valid path
-    assert message.body.objectName
-    _ = Path(message.body.objectName)
-    assert message.body.datasetId == task.constants.dataset_id
-    assert message.body.conversationId == str(task.recipe_run_id)
-    assert message.body.incrementDatasetCatalogReceiptCount is True

{dkist_processing_common-11.9.0rc1 → dkist_processing_common-11.9.1rc1}/dkist_processing_common/tests/test_submit_dataset_metadata.py

@@ -95,13 +95,10 @@ def test_submit_dataset_metadata(
     mocker.patch(
         "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
     )
-    # intercept these two GraphQLClient calls so they can be confirmed
+    # intercept this GraphQLClient call so it can be confirmed
     mocked_metadata_store_add_dataset_receipt_account = mocker.patch.object(
         metadata_store.MetadataStoreMixin, "metadata_store_add_dataset_receipt_account"
     )
-    mocked_metadata_store_add_quality_data = mocker.patch.object(
-        metadata_store.MetadataStoreMixin, "metadata_store_add_quality_data"
-    )
     task = submit_dataset_metadata_task
 
     # When
@@ -109,4 +106,3 @@ def test_submit_dataset_metadata(
 
     # Then
     mocked_metadata_store_add_dataset_receipt_account.assert_called_once()
-    mocked_metadata_store_add_quality_data.assert_called_once()

{dkist_processing_common-11.9.0rc1 → dkist_processing_common-11.9.1rc1}/dkist_processing_common/tests/test_trial_output_data.py

@@ -158,13 +158,12 @@ def complete_trial_output_task(
         task.write(asdf_file_obj, relative_path=asdf_file_name, tags=[Tag.output(), Tag.asdf()])
 
         # Write quality data
-        # quality data is not tagged as OUTPUT
         quality_data_obj = uuid4().hex.encode("utf8")
         quality_data_name = "quality_data.json"
         task.write(
             quality_data_obj,
             relative_path=quality_data_name,
-            tags=Tag.quality_data(),
+            tags=[Tag.output(), Tag.quality_data()],
         )
 
         # Write a quality report file

{dkist_processing_common-11.9.0rc1 → dkist_processing_common-11.9.1rc1}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 11.9.0rc1
+Version: 11.9.1rc1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -57,9 +57,9 @@ Requires-Dist: pytest; extra == "docs"
 Requires-Dist: towncrier<22.12.0; extra == "docs"
 Requires-Dist: dkist-sphinx-theme; extra == "docs"
 Provides-Extra: inventory
-Requires-Dist: dkist-inventory<2.0,>=1.11.1; extra == "inventory"
+Requires-Dist: dkist-inventory==1.11.2; extra == "inventory"
 Provides-Extra: asdf
-Requires-Dist: dkist-inventory[asdf]<2.0,>=1.11.1; extra == "asdf"
+Requires-Dist: dkist-inventory[asdf]==1.11.2; extra == "asdf"
 Provides-Extra: quality
 Requires-Dist: dkist-quality<3.0,>=2.0.0; extra == "quality"
 

{dkist_processing_common-11.9.0rc1 → dkist_processing_common-11.9.1rc1}/dkist_processing_common.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ README.rst
 bitbucket-pipelines.yml
 pyproject.toml
 changelog/.gitempty
+changelog/276.feature.rst
 dkist_processing_common/__init__.py
 dkist_processing_common/config.py
 dkist_processing_common/manual.py

{dkist_processing_common-11.9.0rc1 → dkist_processing_common-11.9.1rc1}/dkist_processing_common.egg-info/requires.txt

@@ -24,7 +24,7 @@ sunpy>=3.0.0
 talus==1.3.4
 
 [asdf]
-dkist-inventory[asdf]<2.0,>=1.11.1
+dkist-inventory[asdf]==1.11.2
 
 [docs]
 sphinx
@@ -36,7 +36,7 @@ towncrier<22.12.0
 dkist-sphinx-theme
 
 [inventory]
-dkist-inventory<2.0,>=1.11.1
+dkist-inventory==1.11.2
 
 [quality]
 dkist-quality<3.0,>=2.0.0

{dkist_processing_common-11.9.0rc1 → dkist_processing_common-11.9.1rc1}/pyproject.toml

@@ -76,10 +76,10 @@ docs = [
   "dkist-sphinx-theme",
 ]
 inventory = [
-  "dkist-inventory >= 1.11.1, <2.0",
+  "dkist-inventory == 1.11.2",
 ]
 asdf = [
-  "dkist-inventory[asdf] >= 1.11.1, <2.0",
+  "dkist-inventory[asdf] == 1.11.2",
 ]
 quality = [
   "dkist-quality >= 2.0.0, <3.0",