dkist-processing-common 6.2.1rc2__tar.gz → 6.2.2__tar.gz

{dkist-processing-common-6.2.1rc2 → dkist-processing-common-6.2.2}/CHANGELOG.rst

@@ -1,3 +1,24 @@
+v6.2.2 (2024-05-07)
+===================
+
+Features
+--------
+
+- Add the ability to create a quality report from a trial workflow. (`#185 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/185>`__)
+
+
+v6.2.1 (2024-05-01)
+===================
+
+Misc
+----
+
+- Change filenames of browse movie and quality report to free up namespace for other future files. (`#124 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/124>`__)
+- Trial framework asdf filenames match production run asdf filenames. (`#186 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/186>`__)
+- Capture tracing data for rollback calls to enhance observability. (`#187 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/187>`__)
+- Update legacy type hinting. (`#188 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/188>`__)
+
+
 v6.1.2 (2024-04-12)
 ===================
 

{dkist-processing-common-6.2.1rc2 → dkist-processing-common-6.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-common
-Version: 6.2.1rc2
+Version: 6.2.2
 Summary: Common task classes used by the DKIST Science Data Processing pipelines to process DKIST data.
 Home-page: https://bitbucket.org/dkistdc/dkist-processing-common/src/main/
 Author: NSO / AURA
@@ -15,6 +15,7 @@ Provides-Extra: test
 Provides-Extra: docs
 Provides-Extra: inventory
 Provides-Extra: asdf
+Provides-Extra: quality
 
 dkist-processing-common
 =======================

dkist-processing-common-6.2.2/dkist_processing_common/codecs/quality.py

@@ -0,0 +1,88 @@
+"""Encoder/decoders for writing and reading quality data."""
+import json
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from dkist_processing_common.codecs.json import json_decoder
+from dkist_processing_common.codecs.json import json_encoder
+
+
+logger = logging.getLogger(__name__)
+
+
+class QualityDataEncoder(json.JSONEncoder):
+    """A JSON encoder for the quality data."""
+
+    def __init__(self, **dumps_kwargs):
+        # Raise a ValueError for NaN and Infinity.
+        dumps_kwargs["allow_nan"] = False
+        super().__init__(**dumps_kwargs)
+
+    def default(self, obj) -> dict:
+        """Encode datetime as dict of iso formatted strings.  JSONEncoder will later render the dict as a string."""
+        if isinstance(obj, datetime):
+            return {"iso_date": obj.isoformat("T")}
+        return super().default(obj)
+
+
+def quality_data_encoder(
+    data: Any, encoding: str = "utf-8", errors="strict", **dumps_kwargs
+) -> bytes:
+    """Convert quality data to bytes by encoding as JSON."""
+    # This encoder is for the QualityDataEncoder class
+    if cls := dumps_kwargs.pop("cls", None):
+        logger.info(
+            f"Ignoring {cls=}.  Default JSONEncoder for quality_data_encoder is QualityDataEncoder."
+        )
+    # allow_nan is per QualityDataEncoder class
+    if allow_nan := dumps_kwargs.pop("allow_nan", None):
+        logger.info(f"Ignoring {allow_nan=} for quality_data_encoder.")
+
+    return json_encoder(
+        data, encoding=encoding, errors=errors, cls=QualityDataEncoder, **dumps_kwargs
+    )
+
+
+def quality_data_hook(obj: dict):
+    """
+    Decode iso date dict.
+
+    Convert object being json decoded into a datetime object if in the format `{"iso_date":"<iso formatted string>"}`
+    like those produced by QualityDataEncoder.
+    This is the same as datetime_json_object_hook in dkist-quality.
+    :param obj: dict of the object being json decoded
+    :return: datetime object
+    """
+    # extract date string if present in the object dict
+    iso_date = obj.get("iso_date")
+    if iso_date is not None:
+        return datetime.fromisoformat(iso_date)
+    # iso_date not found - not covered by this hook
+    return obj
+
+
+def quality_data_decoder(
+    path: Path, encoding: str = "utf-8", errors="strict", **loads_kwargs
+) -> Any:
+    """Read a file as JSON and return the decoded objects."""
+    if object_hook := loads_kwargs.pop("object_hook", None):
+        logger.info(f"Ignoring {object_hook=} for quality_data_decoder.")
+
+    return json_decoder(
+        path, encoding=encoding, errors=errors, object_hook=quality_data_hook, **loads_kwargs
+    )
+
+
+class QualityValueEncoder(json.JSONEncoder):
+    """A JSON encoder applied to the quality metrics."""
+
+    def default(self, obj: Any) -> Any:
+        """Implement an encoder that correctly handles numpy float32 data."""
+        # np.float32 is only for single values. Even an array of np.float32 objects is a np.ndarray
+        if isinstance(obj, np.float32):
+            return float(obj)
+        return super().default(obj)

{dkist-processing-common-6.2.1rc2 → dkist-processing-common-6.2.2}/dkist_processing_common/models/tags.py

@@ -30,8 +30,12 @@ class StemName(str, Enum):
     parameter = "PARAMETER"
     workflow_task = "WORKFLOW_TASK"
     debug = "DEBUG"
+    # The QUALITY_DATA is the json data that normally gets stored to the remote (metadata store) database.
+    quality_data = "QUALITY_DATA"
+    # For trial workflows.
     dataset_inventory = "DATASET_INVENTORY"
     asdf = "ASDF"
+    quality_report = "QUALITY_REPORT"
 
 
 class Tag:
@@ -173,6 +177,16 @@ class Tag:
         """Return a debug tag."""
         return cls.format_tag(StemName.debug)
 
+    @classmethod
+    def quality_data(cls) -> str:
+        """Tags the quality data that normally gets stored to the remote database."""
+        return cls.format_tag(StemName.quality_data.value)
+
+    @classmethod
+    def quality_report(cls) -> str:
+        """Tags the quality report .pdf that gets stored to the file system for trial workflows."""
+        return cls.format_tag(StemName.quality_report.value)
+
     # Task type tags
     @classmethod
     def task_observe(cls) -> str:

{dkist-processing-common-6.2.1rc2 → dkist-processing-common-6.2.2}/dkist_processing_common/tasks/base.py

@@ -1,6 +1,7 @@
 """Wrappers for all workflow tasks."""
 import json
 import logging
+import re
 from abc import ABC
 from pathlib import Path
 from typing import Any
@@ -279,7 +280,8 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
         sorted_remaining_tags = sorted(copied_tags)
         filename_parts += sorted_remaining_tags
 
-        dash_separated_parts = [t.replace("_", "-") for t in filename_parts]
+        # replace spaces and underscores with dashes - dynamic part (e.g. polcal `Beam 1` label) may include spaces
+        dash_separated_parts = [re.sub("[ _]", "-", t) for t in filename_parts]
 
         base_filename = "_".join(dash_separated_parts)
         base_filename_counter = str(self.filename_counter.increment(base_filename))

{dkist-processing-common-6.2.1rc2 → dkist-processing-common-6.2.2}/dkist_processing_common/tasks/l1_output_data.py

@@ -1,31 +1,36 @@
 """Task(s) for the transfer and publishing of L1 data from a production run of a processing pipeline."""
 import logging
+from abc import ABC
+from itertools import chain
 from pathlib import Path
 from typing import Iterable
 
+from dkist_processing_common.codecs.quality import quality_data_decoder
+from dkist_processing_common.codecs.quality import quality_data_encoder
 from dkist_processing_common.models.message import CatalogFrameMessage
 from dkist_processing_common.models.message import CatalogObjectMessage
 from dkist_processing_common.models.message import CreateQualityReportMessage
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.mixin.globus import GlobusMixin
 from dkist_processing_common.tasks.mixin.interservice_bus import InterserviceBusMixin
+from dkist_processing_common.tasks.mixin.quality import QualityMixin
+from dkist_processing_common.tasks.output_data_base import OutputDataBase
+from dkist_processing_common.tasks.output_data_base import TransferDataBase
 
 
 __all__ = [
-    "AddDatasetReceiptAccount",
-    "PublishCatalogAndQualityMessages",
-    "TransferL1Data",
     "L1OutputDataBase",
-    "SubmitQuality",
+    "TransferL1Data",
+    "AssembleQualityData",
+    "SubmitDatasetMetadata",
+    "PublishCatalogAndQualityMessages",
 ]
 
-from dkist_processing_common.tasks.mixin.quality import QualityMixin
-from dkist_processing_common.tasks.output_data_base import OutputDataBase, TransferDataBase
 
 logger = logging.getLogger(__name__)
 
 
-class L1OutputDataBase(OutputDataBase, QualityMixin):
+class L1OutputDataBase(OutputDataBase, ABC):
     """Subclass of OutputDataBase which encapsulates common level 1 output data methods."""
 
     @property
@@ -94,6 +99,72 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
         )
 
 
+class AssembleQualityData(L1OutputDataBase, QualityMixin):
+    """Assemble quality data from the various quality metrics."""
+
+    @property
+    def polcal_label_list(self) -> list[str] | None:
+        """Return the list of labels to look for when building polcal metrics.
+
+        If no labels are specified then no polcal metrics will be built.
+        """
+        return None
+
+    def run(self):
+        """Run method for the task."""
+        with self.apm_processing_step("Assembling quality data"):
+            quality_data = self.quality_assemble_data(polcal_label_list=self.polcal_label_list)
+
+        with self.apm_writing_step(
+            f"Saving quality data with {len(quality_data)} metrics to the file system"
+        ):
+            self.write(
+                quality_data,
+                tags=Tag.quality_data(),
+                encoder=quality_data_encoder,
+                relative_path=f"{self.constants.dataset_id}_quality_data.json",
+            )
+
+
+class SubmitDatasetMetadata(L1OutputDataBase):
+    """
+    Add quality data and receipt account to the metadata store.
+
+    Add the quality data to the Quality database.
+    Add a Dataset Receipt Account record to Processing Support for use by the Dataset Catalog Locker.
+    Adds the number of files created during the calibration processing to the Processing Support table
+    for use by the Dataset Catalog Locker.
+    """
+
+    def run(self) -> None:
+        """Run method for this task."""
+        with self.apm_writing_step(f"Storing quality data to metadata store"):
+            # each quality_data file is a list - this will combine the elements of multiple lists into a single list
+            quality_data = list(
+                chain.from_iterable(
+                    self.read(tags=Tag.quality_data(), decoder=quality_data_decoder)
+                )
+            )
+            self.metadata_store_add_quality_data(
+                dataset_id=self.constants.dataset_id, quality_data=quality_data
+            )
+        with self.apm_processing_step("Count Expected Outputs"):
+            dataset_id = self.constants.dataset_id
+            expected_object_count = self.count(tags=Tag.output())
+            if self.dataset_has_quality_report:
+                expected_object_count += 1
+        logger.info(
+            f"Adding Dataset Receipt Account: "
+            f"{dataset_id=}, {expected_object_count=}, recipe_run_id={self.recipe_run_id}"
+        )
+        with self.apm_task_step(
+            f"Add Dataset Receipt Account: {dataset_id = }, {expected_object_count = }"
+        ):
+            self.metadata_store_add_dataset_receipt_account(
+                dataset_id=dataset_id, expected_object_count=expected_object_count
+            )
+
+
 class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
     """Task class for publishing Catalog and Quality Messages."""
 
@@ -162,7 +233,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         )
 
     def run(self) -> None:
-        """Run method for this trask."""
+        """Run method for this task."""
         with self.apm_task_step("Gather output data"):
             frames = self.read(tags=self.output_frame_tags)
             movies = self.read(tags=[Tag.output(), Tag.movie()])
@@ -179,51 +250,3 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
             f"Publish messages: {frame_message_count = }, {object_message_count = }, {dataset_has_quality_report = }"
         ):
             self.interservice_bus_publish(messages=messages)
-
-
-class AddDatasetReceiptAccount(L1OutputDataBase):
-    """
-    Add a Dataset Receipt Account record to Processing Support for use by the Dataset Catalog Locker.
-
-    Adds the number of files created during the calibration processing to the Processing Support table
-    for use by the Dataset Catalog Locker.
-    """
-
-    def run(self) -> None:
-        """Run method for this task."""
-        with self.apm_processing_step("Count Expected Outputs"):
-            dataset_id = self.constants.dataset_id
-            expected_object_count = self.count(tags=Tag.output())
-            if self.dataset_has_quality_report:
-                expected_object_count += 1
-        logger.info(
-            f"Adding Dataset Receipt Account: "
-            f"{dataset_id=}, {expected_object_count=}, recipe_run_id={self.recipe_run_id}"
-        )
-        with self.apm_task_step(
-            f"Add Dataset Receipt Account: {dataset_id = }, {expected_object_count = }"
-        ):
-            self.metadata_store_add_dataset_receipt_account(
-                dataset_id=dataset_id, expected_object_count=expected_object_count
-            )
-
-
-class SubmitQuality(L1OutputDataBase, QualityMixin):
-    """Task class for submitting the quality report to the metadata store."""
-
-    @property
-    def polcal_label_list(self) -> list[str] | None:
-        """Return the list of labels to look for when building polcal metrics.
-
-        If no labels are specified then no polcal metrics will be built.
-        """
-        return None
-
-    def run(self):
-        """Run method for the task."""
-        with self.apm_processing_step("Building quality report"):
-            report = self.quality_build_report(polcal_label_list=self.polcal_label_list)
-        with self.apm_task_step(f"Submitting quality report: report section count = {len(report)}"):
-            self.metadata_store_add_quality_report(
-                dataset_id=self.constants.dataset_id, quality_report=report
-            )

{dkist-processing-common-6.2.1rc2 → dkist-processing-common-6.2.2}/dkist_processing_common/tasks/mixin/metadata_store.py

@@ -5,6 +5,7 @@ from functools import cached_property
 
 from dkist_processing_common._util.config import service_configuration
 from dkist_processing_common._util.graphql import GraphQLClient
+from dkist_processing_common.codecs.quality import QualityDataEncoder
 from dkist_processing_common.models.graphql import DatasetCatalogReceiptAccountMutation
 from dkist_processing_common.models.graphql import DatasetCatalogReceiptAccountResponse
 from dkist_processing_common.models.graphql import InputDatasetPartResponse
@@ -21,7 +22,6 @@ from dkist_processing_common.models.graphql import RecipeRunResponse
 from dkist_processing_common.models.graphql import RecipeRunStatusMutation
 from dkist_processing_common.models.graphql import RecipeRunStatusQuery
 from dkist_processing_common.models.graphql import RecipeRunStatusResponse
-from dkist_processing_common.models.quality_json_encoders import QualityReportEncoder
 
 
 logger = logging.getLogger(__name__)
@@ -78,10 +78,10 @@ class MetadataStoreMixin:
             mutation_response_cls=RecipeRunProvenanceResponse,
         )
 
-    def metadata_store_add_quality_report(self, dataset_id: str, quality_report: list[dict]):
-        """Add the quality report to the metadata-store."""
-        quality_report_json = json.dumps(quality_report, cls=QualityReportEncoder, allow_nan=False)
-        params = QualityReportMutation(datasetId=dataset_id, qualityReport=quality_report_json)
+    def metadata_store_add_quality_data(self, dataset_id: str, quality_data: list[dict]):
+        """Add the quality data to the metadata-store."""
+        quality_data_json = json.dumps(quality_data, cls=QualityDataEncoder)
+        params = QualityReportMutation(datasetId=dataset_id, qualityReport=quality_data_json)
         self.metadata_store_client.execute_gql_mutation(
             mutation_base="createQualityReport",
             mutation_parameters=params,

{dkist-processing-common-6.2.1rc2 → dkist-processing-common-6.2.2}/dkist_processing_common/tasks/mixin/quality/_base.py

@@ -4,7 +4,7 @@ from typing import Iterable
 import numpy as np
 
 from dkist_processing_common.codecs.json import json_encoder
-from dkist_processing_common.models.quality_json_encoders import QualityValueEncoder
+from dkist_processing_common.codecs.quality import QualityValueEncoder
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.mixin.quality._metrics import _PolcalQualityMixin
 from dkist_processing_common.tasks.mixin.quality._metrics import _SimplePlotQualityMixin
@@ -17,8 +17,8 @@ class QualityMixin(
 ):
     """Mixin class supporting the generation of the quality reports."""
 
-    def quality_build_report(self, polcal_label_list: list[str] | None = None) -> list[dict]:
-        """Build the quality report by checking for the existence of data for each metric."""
+    def quality_assemble_data(self, polcal_label_list: list[str] | None = None) -> list[dict]:
+        """Assemble the quality data by checking for the existence of each metric."""
         report = []
         report += self.quality_task_independent_metrics()
         report += self.quality_task_dependent_metrics()

{dkist-processing-common-6.2.1rc2 → dkist-processing-common-6.2.2}/dkist_processing_common/tasks/quality_metrics.py

@@ -84,7 +84,9 @@ class QualityL0Metrics(WorkflowTaskBase, QualityMixin, ABC):
                         # We grab the task name
                         tags = self.tags(path)
                         task_type = [
-                            t.replace(f"{StemName.task.value}_", "") for t in tags if "TASK" in t
+                            t.replace(f"{StemName.task.value}_", "")
+                            for t in tags
+                            if t.startswith("TASK")
                         ][0]
 
                         if (

{dkist-processing-common-6.2.1rc2 → dkist-processing-common-6.2.2}/dkist_processing_common/tasks/trial_catalog.py

@@ -2,6 +2,7 @@
 import logging
 from datetime import datetime
 from importlib import resources
+from itertools import chain
 from pathlib import Path
 from typing import Generator
 from uuid import uuid4
@@ -10,13 +11,14 @@ from dkist_processing_common.codecs.asdf import asdf_encoder
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.json import json_encoder
 from dkist_processing_common.codecs.path import path_decoder
+from dkist_processing_common.codecs.quality import quality_data_decoder
 from dkist_processing_common.models.fits_access import FitsAccessBase
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.output_data_base import OutputDataBase
 
 logger = logging.getLogger(__name__)
 
-__all__ = ["CreateTrialDatasetInventory", "CreateTrialAsdf"]
+__all__ = ["CreateTrialDatasetInventory", "CreateTrialAsdf", "CreateTrialQualityReport"]
 
 
 # Capture condition of dkist-processing-common[inventory] install
@@ -39,6 +41,16 @@ try:
 except ModuleNotFoundError:
     pass
 
+# Verify dkist-quality is installed
+QUALITY_EXTRA_INSTALLED = False
+try:
+    from dkist_quality.report import format_report
+    from dkist_quality.report import ReportFormattingException
+
+    QUALITY_EXTRA_INSTALLED = True
+except ModuleNotFoundError:
+    pass
+
 
 class CreateTrialDatasetInventory(OutputDataBase):
     """
@@ -143,3 +155,44 @@ class CreateTrialAsdf(OutputDataBase):
                     ),
                     custom_schema=schema_path.as_posix(),
                 )
+
+
+class CreateTrialQualityReport(OutputDataBase):
+    """
+    Task for use in Trial workflows to generate the quality report for the dataset.
+
+    Warning: This task requires the dkist-quality package.
+    """
+
+    def pre_run(self) -> None:
+        """Require the dkist-quality package be installed."""
+        if not QUALITY_EXTRA_INSTALLED:
+            raise ModuleNotFoundError(
+                f"{self.__class__.__name__} Task requires the dkist-quality package "
+                f"(e.g. via a 'quality' pip_extra on dkist_processing_core.Workflow().add_node())"
+                f" but the required dependencies were not found."
+            )
+
+    def run(self) -> None:
+        """Generate the quality report for the dataset."""
+        self.create_trial_quality_report()
+
+    def create_trial_quality_report(self) -> None:
+        """Generate a trial quality report in pdf format and save to the file system for future upload."""
+        with self.apm_processing_step(f"Building the trial quality report"):
+            # each quality_data file is a list - this will combine the elements of multiple lists into a single list
+            quality_data = list(
+                chain.from_iterable(
+                    self.read(tags=Tag.quality_data(), decoder=quality_data_decoder)
+                )
+            )
+            quality_report = format_report(
+                report_data=quality_data, dataset_id=self.constants.dataset_id
+            )
+
+        with self.apm_writing_step(f"Saving the trial quality report to the file system"):
+            self.write(
+                quality_report,
+                tags=[Tag.output(), Tag.quality_report()],
+                relative_path=f"{self.constants.dataset_id}_quality_report.pdf",
+            )

{dkist-processing-common-6.2.1rc2 → dkist-processing-common-6.2.2}/dkist_processing_common/tasks/trial_output_data.py

@@ -93,6 +93,20 @@ class TransferTrialDataBase(TransferDataBase, GlobusMixin, ABC):
             "trial_transfer_output_asdf", True
         )
 
+    @property
+    def output_quality_data_switch(self) -> bool:
+        """Switch to turn on/off the transfer of the quality data to the trial location."""
+        return self.metadata_store_recipe_run_configuration().get(
+            "trial_transfer_output_quality_data", True
+        )
+
+    @property
+    def output_quality_report_switch(self) -> bool:
+        """Switch to turn on/off the transfer of the quality report to the trial location."""
+        return self.metadata_store_recipe_run_configuration().get(
+            "trial_transfer_output_quality_report", True
+        )
+
     @property
     def specific_frame_tag_lists(self) -> list:
         """Return list of tag lists that define specific files we want to transfer to the trial location."""
@@ -161,6 +175,19 @@ class TransferTrialDataBase(TransferDataBase, GlobusMixin, ABC):
         )
         return transfer_items
 
+    def build_output_quality_data_transfer_list(self) -> list[GlobusTransferItem]:
+        """Build a transfer list containing all files tagged with OUTPUT and QUALITY_DATA."""
+        # quality data is not tagged as OUTPUT
+        transfer_items = self.build_transfer_list_from_tag_lists(tag_lists=[Tag.quality_data()])
+        return transfer_items
+
+    def build_output_quality_report_transfer_list(self) -> list[GlobusTransferItem]:
+        """Build a transfer list containing all files tagged with OUTPUT and QUALITY_REPORT."""
+        transfer_items = self.build_transfer_list_from_tag_lists(
+            tag_lists=[Tag.output(), Tag.quality_report()]
+        )
+        return transfer_items
+
     @property
     def intermediate_task_names(self) -> list[str]:
         """List specifying which TASK types to build when selecting INTERMEDIATE frames."""