dkist-processing-common 10.5.4__py3-none-any.whl → 12.1.0rc1__py3-none-any.whl

dkist_processing_common/tasks/quality_metrics.py

@@ -1,4 +1,5 @@
 """Classes to support the generation of quality metrics for the calibrated data."""
+
 import logging
 from dataclasses import dataclass
 from dataclasses import field
@@ -18,7 +19,6 @@ from dkist_processing_common.parsers.quality import L1QualityFitsAccess
 from dkist_processing_common.tasks.base import WorkflowTaskBase
 from dkist_processing_common.tasks.mixin.quality import QualityMixin
 
-
 __all__ = ["QualityL1Metrics", "QualityL0Metrics"]
 
 
@@ -100,10 +100,10 @@ class QualityL0Metrics(WorkflowTaskBase, QualityMixin):
         `quality_task_types` properties, respectively.
         """
         modstate_list = self.modstate_list if self.modstate_list is not None else [None]
-        with self.apm_task_step("Computing L0 Quality Metrics"):
+        with self.telemetry_span("Computing L0 Quality Metrics"):
             quality_data_list = []
             for task_type in self.quality_task_types:
-                with self.apm_processing_step(f"Working on {task_type = }"):
+                with self.telemetry_span(f"Working on {task_type = }"):
                     for modstate in modstate_list:
                         paths = self.get_paths_for_modstate_and_task(modstate, task_type)
                         quality_data = self.calculate_l0_metrics(
@@ -112,7 +112,7 @@ class QualityL0Metrics(WorkflowTaskBase, QualityMixin):
                         quality_data.modstate = modstate
                         quality_data_list.append(quality_data)
 
-        with self.apm_writing_step("Saving metrics to disk"):
+        with self.telemetry_span("Saving metrics to disk"):
             for quality_data in quality_data_list:
                 if quality_data.has_values:
                     self.save_quality_data(quality_data, modstate=quality_data.modstate)
@@ -296,21 +296,21 @@ class QualityL1Metrics(WorkflowTaskBase, QualityMixin):
             L1Metric(storage_method=self.quality_store_health_status, value_source="health_status"),
             L1Metric(
                 storage_method=self.quality_store_ao_status_and_fried_parameter,
-                value_source=["ao_status", "fried_parameter"],
+                value_source=["ao_status", "fried_parameter", "num_out_of_bounds_ao_values"],
             ),
         ]
 
-        with self.apm_task_step("Reading L1 frames"):
+        with self.telemetry_span("Reading L1 frames"):
             paths = list(self.read(tags=[Tag.calibrated(), Tag.frame()]))
 
-        with self.apm_task_step("Calculating L1 quality metrics"):
+        with self.telemetry_span("Calculating L1 quality metrics"):
             for metric in metrics:
-                with self.apm_processing_step(f"Calculating L1 metric {metric.value_source}"):
+                with self.telemetry_span(f"Calculating L1 metric {metric.value_source}"):
                     for path in paths:
                         frame = L1QualityFitsAccess.from_path(path)
                         metric.append_value(frame=frame)
 
-        with self.apm_task_step("Sending lists for storage"):
+        with self.telemetry_span("Sending lists for storage"):
             for metric in metrics:
                 if metric.has_values:
                     metric.store_metric()

dkist_processing_common/tasks/teardown.py

@@ -1,11 +1,11 @@
 """Task(s) for the clean up tasks at the conclusion of a processing pipeline."""
+
 import logging
 from abc import ABC
 from abc import abstractmethod
 
 from dkist_processing_common.tasks.base import WorkflowTaskBase
 
-
 __all__ = ["Teardown", "TrialTeardown"]
 
 
@@ -22,15 +22,15 @@ class TeardownBase(WorkflowTaskBase, ABC):
     @property
     def teardown_enabled(self) -> bool:
         """Recipe run configuration indicating if data should be removed at the end of a run."""
-        return self.metadata_store_recipe_run_configuration().get("teardown_enabled", True)
+        return self.metadata_store_recipe_run.configuration.teardown_enabled
 
     def run(self) -> None:
         """Run method for Teardown class."""
-        with self.apm_task_step("Change recipe run status"):
+        with self.telemetry_span("Change recipe run status"):
             self.change_recipe_run_status_to_success()
 
         if not self.teardown_enabled:
-            with self.apm_task_step(f"Skip Teardown"):
+            with self.telemetry_span(f"Skip Teardown"):
                 return
 
         logger.info(f"Removing data and tags for recipe run {self.recipe_run_id}")
@@ -43,13 +43,13 @@ class TeardownBase(WorkflowTaskBase, ABC):
 
     def teardown(self):
         """Purge all constants and files/tags in scratch."""
-        with self.apm_task_step("Remove Data and Tags"):
+        with self.telemetry_span("Remove Data and Tags"):
             self.scratch.purge()
 
-        with self.apm_task_step("Remove File Counters"):
+        with self.telemetry_span("Remove File Counters"):
             self.filename_counter.purge()
 
-        with self.apm_task_step("Remove Constants"):
+        with self.telemetry_span("Remove Constants"):
             self.constants._purge()
 
 

dkist_processing_common/tasks/transfer_input_data.py

@@ -1,30 +1,45 @@
 """Task(s) for the transfer in of data sources for a processing pipeline."""
+
 import logging
 from pathlib import Path
 
+from dkist_processing_common.codecs.basemodel import basemodel_decoder
+from dkist_processing_common.codecs.basemodel import basemodel_encoder
+from dkist_processing_common.models.input_dataset import InputDatasetObject
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.base import WorkflowTaskBase
 from dkist_processing_common.tasks.mixin.globus import GlobusMixin
 from dkist_processing_common.tasks.mixin.globus import GlobusTransferItem
-from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetMixin
-from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetObject
 
 __all__ = ["TransferL0Data"]
 
 logger = logging.getLogger(__name__)
 
 
-class TransferL0Data(WorkflowTaskBase, GlobusMixin, InputDatasetMixin):
+class TransferL0Data(WorkflowTaskBase, GlobusMixin):
     """Transfers Level 0 data and required parameter files to the scratch store."""
 
     def download_input_dataset(self):
-        """Get the input dataset document parts and save it to scratch with the appropriate tags."""
-        if doc := self.metadata_store_input_dataset_observe_frames_part_document:
-            self.write(doc.encode("utf-8"), tags=Tag.input_dataset_observe_frames())
-        if doc := self.metadata_store_input_dataset_calibration_frames_part_document:
-            self.write(doc.encode("utf-8"), tags=Tag.input_dataset_calibration_frames())
-        if doc := self.metadata_store_input_dataset_parameters_part_document:
-            self.write(doc.encode("utf-8"), tags=Tag.input_dataset_parameters())
+        """Write the input dataset part documents to scratch with appropriate tags."""
+        if observe_frames_part := self.metadata_store_input_dataset_observe_frames:
+            doc = observe_frames_part.inputDatasetPartDocument
+            self.write(data=doc, tags=Tag.input_dataset_observe_frames(), encoder=basemodel_encoder)
+        if calibration_frames_part := self.metadata_store_input_dataset_calibration_frames:
+            doc = calibration_frames_part.inputDatasetPartDocument
+            self.write(
+                data=doc, tags=Tag.input_dataset_calibration_frames(), encoder=basemodel_encoder
+            )
+        if parameters_part := self.metadata_store_input_dataset_parameters:
+            doc = parameters_part.inputDatasetPartDocument
+            self.add_file_tags_to_parameters_doc(param_doc=doc)
+            self.write(data=doc, tags=Tag.input_dataset_parameters(), encoder=basemodel_encoder)
+
+    def add_file_tags_to_parameters_doc(self, param_doc: InputDatasetPartDocumentList):
+        """Update the input dataset document with the location of the file parameters."""
+        for doc_item in param_doc.doc_list:
+            for obj in doc_item.input_dataset_objects:
+                obj.tag = Tag.parameter(Path(obj.object_key).name)
 
     def format_transfer_items(
         self, input_dataset_objects: list[InputDatasetObject]
@@ -43,77 +58,60 @@ class TransferL0Data(WorkflowTaskBase, GlobusMixin, InputDatasetMixin):
             )
         return transfer_items
 
-    def format_frame_transfer_items(self) -> list[GlobusTransferItem]:
+    def build_transfer_list(self, doc_tag: str) -> list[InputDatasetObject]:
         """Format the list of frames as transfer items to be used by globus."""
-        return self.format_transfer_items(self.input_dataset_frames)
-
-    def format_parameter_transfer_items(self) -> list[GlobusTransferItem]:
-        """Format the list of parameter objects as transfer items to be used by globus."""
-        return self.format_transfer_items(self.input_dataset_parameter_objects)
-
-    def tag_input_frames(self, transfer_items: list[GlobusTransferItem]) -> None:
-        """
-        Tag all the input files with 'frame' and 'input' tags.
-
-        Parameters
-        ----------
-        transfer_items
-            List of items to be tagged
-
-        Returns
-        -------
-        None
-        """
-        scratch_items = [
-            self.scratch.scratch_base_path / ti.destination_path for ti in transfer_items
-        ]
-        for si in scratch_items:
-            self.tag(si, tags=[Tag.input(), Tag.frame()])
-
-    def tag_parameter_objects(self, transfer_items: list[GlobusTransferItem]) -> None:
-        """
-        Tag all the parameter files with 'parameter'.
-
-        Parameters
-        ----------
-        transfer_items
-            List of items to be tagged
-
-        Returns
-        -------
-        None
-        """
-        scratch_items = [
-            self.scratch.scratch_base_path / ti.destination_path for ti in transfer_items
-        ]
-        for si in scratch_items:
-            self.tag(si, tags=[Tag.parameter(si.name)])
+        doc = next(
+            self.read(tags=doc_tag, decoder=basemodel_decoder, model=InputDatasetPartDocumentList),
+            None,
+        )
+        doc_list = doc.doc_list if doc else []
+        input_dataset_objects = []
+        for doc_item in doc_list:
+            input_dataset_objects += doc_item.input_dataset_objects
+        return input_dataset_objects
+
+    def tag_transfer_objects(self, input_dataset_objects: list[InputDatasetObject]) -> None:
+        """Tag all the transferred input files."""
+        for obj in input_dataset_objects:
+            obj_path = self.scratch.absolute_path(obj.object_key)
+            if obj.tag:
+                self.tag(obj_path, tags=obj.tag)
+            else:
+                self.tag(obj_path, tags=[Tag.input(), Tag.frame()])
+        logger.info(f"Tagged {len(input_dataset_objects)} input dataset objects in scratch")
 
     def run(self) -> None:
         """Execute the data transfer."""
-        with self.apm_task_step("Change Status to InProgress"):
+        with self.telemetry_span("Change Status to InProgress"):
             self.metadata_store_change_recipe_run_to_inprogress()
 
-        with self.apm_task_step("Download Input Dataset"):
+        with self.telemetry_span("Download Input Dataset Documents"):
             self.download_input_dataset()
 
-        with self.apm_task_step("Format Frame Transfer Items"):
-            frame_transfer_items = self.format_frame_transfer_items()
-            if not frame_transfer_items:
-                raise ValueError("No input dataset frames found")
-
-        with self.apm_task_step("Format Parameter Transfer Items"):
-            parameter_transfer_items = self.format_parameter_transfer_items()
+        with self.telemetry_span("Build Input Dataset Transfer List"):
+            observe_transfer_objects = self.build_transfer_list(
+                doc_tag=Tag.input_dataset_observe_frames()
+            )
+            calibration_transfer_objects = self.build_transfer_list(
+                doc_tag=Tag.input_dataset_calibration_frames()
+            )
+            parameter_transfer_objects = self.build_transfer_list(
+                doc_tag=Tag.input_dataset_parameters()
+            )
+            transfer_objects = (
+                observe_transfer_objects + calibration_transfer_objects + parameter_transfer_objects
+            )
+            if len(observe_transfer_objects + calibration_transfer_objects) == 0:
+                raise ValueError("No input dataset frames found to transfer")
 
-        with self.apm_task_step("Transfer Input Frames and Parameter Files via Globus"):
+        with self.telemetry_span("Transfer Input Frames and Parameter Files via Globus"):
             self.globus_transfer_object_store_to_scratch(
-                transfer_items=frame_transfer_items + parameter_transfer_items,
-                label=f"Transfer Inputs for Recipe Run {self.recipe_run_id}",
+                transfer_items=self.format_transfer_items(input_dataset_objects=transfer_objects),
+                label=f"Transfer Input Objects for Recipe Run {self.recipe_run_id}",
             )
 
-        with self.apm_processing_step("Tag Input Frames and Parameter Files"):
-            self.tag_input_frames(transfer_items=frame_transfer_items)
-            self.tag_parameter_objects(transfer_items=parameter_transfer_items)
+        with self.telemetry_span("Tag Input Frames and Parameter Files"):
+            self.tag_transfer_objects(input_dataset_objects=transfer_objects)
 
     def rollback(self):
         """Warn that depending on the progress of the task all data may not be removed because it hadn't been tagged."""

dkist_processing_common/tasks/trial_catalog.py

@@ -1,18 +1,22 @@
 """Tasks to support the generation of downstream artifacts in a trial workflow that wouldn't otherwise produce them."""
-import importlib.resources as resources
+
+import importlib
 import logging
 from datetime import datetime
 from itertools import chain
 from pathlib import Path
+from typing import Any
 from typing import Generator
 from uuid import uuid4
 
-from dkist_processing_common.codecs.asdf import asdf_encoder
+from dkist_processing_common.codecs.asdf import asdf_fileobj_encoder
+from dkist_processing_common.codecs.basemodel import basemodel_decoder
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.json import json_encoder
 from dkist_processing_common.codecs.path import path_decoder
 from dkist_processing_common.codecs.quality import quality_data_decoder
 from dkist_processing_common.models.fits_access import FitsAccessBase
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.output_data_base import OutputDataBase
 
@@ -24,8 +28,9 @@ __all__ = ["CreateTrialDatasetInventory", "CreateTrialAsdf", "CreateTrialQuality
 # Capture condition of dkist-processing-common[inventory] install
 INVENTORY_EXTRA_INSTALLED = False
 try:
-    from dkist_inventory.inventory import generate_inventory_from_frame_inventory
     from dkist_inventory.inventory import generate_asdf_filename
+    from dkist_inventory.inventory import generate_inventory_from_frame_inventory
+    from dkist_inventory.inventory import generate_quality_report_filename
 
     INVENTORY_EXTRA_INSTALLED = True
 except ModuleNotFoundError:
@@ -36,6 +41,7 @@ ASDF_EXTRA_INSTALLED = False
 try:
     import asdf
     from dkist_inventory.asdf_generator import asdf_tree_from_filenames
+    from dkist_inventory.asdf_generator import make_asdf_file_object
 
     ASDF_EXTRA_INSTALLED = True
 except ModuleNotFoundError:
@@ -44,8 +50,8 @@ except ModuleNotFoundError:
 # Verify dkist-quality is installed
 QUALITY_EXTRA_INSTALLED = False
 try:
-    from dkist_quality.report import format_report
     from dkist_quality.report import ReportFormattingException
+    from dkist_quality.report import format_report
 
     QUALITY_EXTRA_INSTALLED = True
 except ModuleNotFoundError:
@@ -95,13 +101,13 @@ class CreateTrialDatasetInventory(OutputDataBase):
 
     def run(self) -> None:
         """Generate a json file simulating the dataset inventory record that would be produced when cataloging the dataset."""
-        with self.apm_task_step("Retrieve output frame headers"):
+        with self.telemetry_span("Retrieve output frame headers"):
             json_headers = list(self.frame_inventories)
-        with self.apm_processing_step("Generate dataset inventory"):
+        with self.telemetry_span("Generate dataset inventory"):
             inventory: dict = generate_inventory_from_frame_inventory(
                 bucket=self.destination_bucket, json_headers=json_headers
             )
-        with self.apm_writing_step("Save dataset inventory file"):
+        with self.telemetry_span("Save dataset inventory file"):
             self.write(
                 inventory,
                 tags=[Tag.output(), Tag.dataset_inventory()],
@@ -136,27 +142,72 @@ class CreateTrialAsdf(OutputDataBase):
 
     def run(self) -> None:
         """Generate an ASDF file simulating the ASDF file that would be produced when cataloging the dataset."""
-        with self.apm_processing_step("Generate ASDF tree"):
+        with self.telemetry_span("Collate input dataset parameters"):
+            parameters = self.parse_input_dataset_parameters()
+
+        with self.telemetry_span("Generate ASDF tree"):
             tree = asdf_tree_from_filenames(
                 filenames=self.absolute_output_frame_paths,
                 hdu=1,  # compressed
                 relative_to=self.scratch.workflow_base_path,
+                parameters=parameters,
             )
-        with self.apm_writing_step("Save ASDF file"):
-            resource = resources.files("dkist.io") / "level_1_dataset_schema.yaml"
-            with resources.as_file(resource) as schema_path:
+
+        trial_history = [
+            (
+                "Written with dkist-processing-common trial ASDF writer",
+                {
+                    "name": "dkist-processing-common",
+                    "author": "DKIST Data Center",
+                    "homepage": "https://bitbucket.org/dkistdc/dkist-processing-common",
+                    "version": importlib.metadata.distribution("dkist-processing-common").version,
+                },
+            )
+        ]
+        with self.telemetry_span("Save ASDF file"):
+            with make_asdf_file_object(tree, extra_history=trial_history) as asdf_obj:
                 self.write(
-                    tree,
+                    asdf_obj,
                     tags=[Tag.output(), Tag.asdf()],
-                    encoder=asdf_encoder,
+                    encoder=asdf_fileobj_encoder,
                     relative_path=generate_asdf_filename(
                         instrument=self.constants.instrument,
                         start_time=datetime.fromisoformat(self.constants.obs_ip_start_time),
                         dataset_id=self.constants.dataset_id,
                     ),
-                    custom_schema=schema_path.as_posix(),
                 )
 
+    def parse_input_dataset_parameters(self) -> list[dict[str, Any]]:
+        """
+        Return the parameters associated with the dataset.
+
+        Returns
+        -------
+        list[dict[str, Any]]
+            A list of dictionaries, each containing a parameter name and its values.
+
+        Raises
+        ------
+        ValueError
+            If there is not exactly one ``InputDatasetPartDocumentList`` found.
+        """
+        part_docs_iter = self.read(
+            tags=Tag.input_dataset_parameters(),
+            decoder=basemodel_decoder,
+            model=InputDatasetPartDocumentList,
+        )
+        docs = list(part_docs_iter)
+
+        if not docs:
+            logger.warning("No parameter list decoded from files")
+            return []
+
+        if len(docs) > 1:
+            raise ValueError(f"Expected 1 parameter list, found {len(docs)}")
+
+        parameters = docs[0].model_dump(by_alias=True).get("doc_list", [])
+        return parameters
+
 
 class CreateTrialQualityReport(OutputDataBase):
     """
@@ -174,13 +225,20 @@ class CreateTrialQualityReport(OutputDataBase):
                 f" but the required dependencies were not found."
             )
 
+        if not INVENTORY_EXTRA_INSTALLED:
+            raise ModuleNotFoundError(
+                f"{self.__class__.__name__} Task requires the dkist-inventory package "
+                f"(e.g. via an 'inventory' pip_extra on dkist_processing_core.Workflow().add_node())"
+                f" but the required dependencies were not found."
+            )
+
     def run(self) -> None:
         """Generate the quality report for the dataset."""
         self.create_trial_quality_report()
 
     def create_trial_quality_report(self) -> None:
         """Generate a trial quality report in pdf format and save to the file system for future upload."""
-        with self.apm_processing_step(f"Building the trial quality report"):
+        with self.telemetry_span(f"Building the trial quality report"):
             # each quality_data file is a list - this will combine the elements of multiple lists into a single list
             quality_data = list(
                 chain.from_iterable(
@@ -191,9 +249,11 @@ class CreateTrialQualityReport(OutputDataBase):
                 report_data=quality_data, dataset_id=self.constants.dataset_id
             )
 
-        with self.apm_writing_step(f"Saving the trial quality report to the file system"):
+        with self.telemetry_span(f"Saving the trial quality report to the file system"):
             self.write(
                 quality_report,
                 tags=[Tag.output(), Tag.quality_report()],
-                relative_path=f"{self.constants.dataset_id}_quality_report.pdf",
+                relative_path=generate_quality_report_filename(
+                    dataset_id=self.constants.dataset_id
+                ),
             )

dkist_processing_common/tasks/trial_output_data.py

@@ -1,4 +1,5 @@
 """Tasks to support transferring an arbitrary collection of files to a customizable post-run location."""
+
 import logging
 from functools import cached_property
 from pathlib import Path
@@ -35,33 +36,31 @@ class TransferTrialData(TransferDataBase, GlobusMixin):
 
     def transfer_objects(self) -> None:
         """Collect transfer items and send them to Globus for transfer."""
-        with self.apm_task_step("Build transfer list"):
+        with self.telemetry_span("Build transfer list"):
             transfer_manifest = self.build_transfer_list()
 
-        with self.apm_task_step("Send transfer manifest to globus"):
+        with self.telemetry_span("Send transfer manifest to globus"):
             self.transfer_all_trial_frames(transfer_manifest)
 
     @cached_property
     def destination_bucket(self) -> str:
-        """Get the destination bucket with a trial default."""
-        return self.metadata_store_recipe_run_configuration().get("destination_bucket", "etc")
+        """Get the destination bucket."""
+        return self.metadata_store_recipe_run.configuration.destination_bucket
 
     @property
     def destination_root_folder(self) -> Path:
         """Format the destination root folder with a value that can be set in the recipe run configuration."""
-        root_name_from_configuration = self.metadata_store_recipe_run_configuration().get(
-            "trial_root_directory_name"
+        root_name_from_config = (
+            self.metadata_store_recipe_run.configuration.trial_root_directory_name
         )
-        root_name = Path(root_name_from_configuration or super().destination_root_folder)
-
+        root_name = Path(root_name_from_config or super().destination_root_folder)
         return root_name
 
     @property
     def destination_folder(self) -> Path:
         """Format the destination folder with a parent that can be set by the recipe run configuration."""
-        dir_name = self.metadata_store_recipe_run_configuration().get(
-            "trial_directory_name"
-        ) or Path(self.constants.dataset_id)
+        dir_name_from_config = self.metadata_store_recipe_run.configuration.trial_directory_name
+        dir_name = dir_name_from_config or Path(self.constants.dataset_id)
         return self.destination_root_folder / dir_name
 
     @property
@@ -71,9 +70,12 @@ class TransferTrialData(TransferDataBase, GlobusMixin):
         Defaults to transferring all product files.  Setting `trial_exclusive_transfer_tag_lists` in the
         recipe run configuration to a list of tag lists will override the default.
         """
-        return self.metadata_store_recipe_run_configuration().get(
-            "trial_exclusive_transfer_tag_lists", self.default_transfer_tag_lists
+        tag_list_from_config = (
+            self.metadata_store_recipe_run.configuration.trial_exclusive_transfer_tag_lists
         )
+        if tag_list_from_config is not None:
+            return tag_list_from_config
+        return self.default_transfer_tag_lists
 
     @property
     def output_frame_tag_list(self) -> list[list[str]]:
@@ -98,7 +100,7 @@ class TransferTrialData(TransferDataBase, GlobusMixin):
         tag_list = []
         tag_list += [[Tag.output(), Tag.dataset_inventory()]]
         tag_list += [[Tag.output(), Tag.asdf()]]
-        tag_list += [[Tag.quality_data()]]  # quality data is not tagged as OUTPUT
+        tag_list += [[Tag.output(), Tag.quality_data()]]
         tag_list += [[Tag.output(), Tag.quality_report()]]
         tag_list += [[Tag.output(), Tag.movie()]]
         return tag_list
@@ -128,9 +130,6 @@ class TransferTrialData(TransferDataBase, GlobusMixin):
         """
         tag_lists = self.transfer_tag_lists
 
-        if not isinstance(tag_lists[0], list):
-            raise ValueError(f"{tag_lists=} must be a list of tag set lists")
-
         transfer_items = []
         for tag_set in tag_lists: