dkist-processing-common 12.0.0rc5__py3-none-any.whl → 12.2.0__py3-none-any.whl

dkist_processing_common/parsers/near_bud.py

@@ -4,15 +4,15 @@ from enum import StrEnum
 from statistics import mean
 from typing import Callable
 
+from dkist_processing_common.models.flower_pot import ListStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
-from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 
 
-class NearFloatBud(Stem):
+class NearFloatBud(ListStem):
     """
-    Pre-made flower that reads a single header key from all files and raises a ValueError if the values are not within a given tolerance.
+    Pre-made `ListStem` that reads a single header key from all files and raises a ValueError if the values are not within a given tolerance.
 
     This is intended for use with floats where the values may be slightly different, but should be the same.
 
@@ -54,26 +54,21 @@ class NearFloatBud(Stem):
         """
         return getattr(fits_obj, self.metadata_key)
 
-    def getter(self, key):
+    def getter(self):
         """
         Get the value for this key and raise an error if the data spans more than the given tolerance.
 
-        Parameters
-        ----------
-        key
-            The input key
         Returns
         -------
         The mean value associated with this input key
         """
-        value_list = list(self.key_to_petal_dict.values())
-        biggest_value = max(value_list)
-        smallest_value = min(value_list)
+        biggest_value = max(self.value_list)
+        smallest_value = min(self.value_list)
         if biggest_value - smallest_value > self.tolerance:
             raise ValueError(
                 f"{self.stem_name} values are not close enough. Max: {biggest_value}, Min: {smallest_value}, Tolerance: {self.tolerance}"
             )
-        return mean(value_list)
+        return mean(self.value_list)
 
 
 class TaskNearFloatBud(NearFloatBud):

dkist_processing_common/parsers/retarder.py

@@ -2,7 +2,9 @@
 
 from dkist_processing_common.models.constants import BudName
 from dkist_processing_common.models.fits_access import MetadataKey
+from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.models.task_name import TaskName
+from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
 
 
@@ -11,13 +13,9 @@ class RetarderNameBud(TaskUniqueBud):
     Bud for determining the name of the retarder used during a polcal Calibration Sequence (CS).
 
     This is *slightly* different than a simple `TaskUniqueBud` because we need to allow for CS steps when the retarder
-    is out of the beam (i.g., "clear"). We do this by forcing the set of header values to be `{clear, RETARDER_NAME}`,
-    where RETARDER_NAME is the value of this Bud.
+    is out of the beam (i.g., "clear"). We do this by returning `SpilledDirt` from the `setter` if the value is "clear".
     """
 
-    # For type-hinting later
-    key_to_petal_dict: dict[str, str]
-
     def __init__(self):
         super().__init__(
             constant_name=BudName.retarder_name,
@@ -25,12 +23,10 @@ class RetarderNameBud(TaskUniqueBud):
             ip_task_types=TaskName.polcal,
         )
 
-    def getter(self, key) -> str:
-        """Get the value for the retarder name and raise an Error if, ignoring "clear", that name is not unique."""
-        value_set = set(self.key_to_petal_dict.values())
-        value_set -= {"clear"}
-        if len(value_set) > 1:
-            raise ValueError(f"Multiple non-clear retarder names found. Names: {value_set}")
+    def setter(self, fits_obj: L0FitsAccess) -> type[SpilledDirt] | str:
+        """Drop the result if the retarder is out of the beam ("clear")."""
+        result = super().setter(fits_obj)
+        if result is not SpilledDirt and result.casefold() == "clear":
+            return SpilledDirt
 
-        raw_retarder_name = value_set.pop()
-        return raw_retarder_name
+        return result

dkist_processing_common/parsers/time.py

@@ -4,15 +4,15 @@ from datetime import datetime
 from datetime import timezone
 from enum import StrEnum
 from typing import Callable
-from typing import Hashable
 from typing import Type
 
 import numpy as np
 
 from dkist_processing_common.models.constants import BudName
 from dkist_processing_common.models.fits_access import MetadataKey
+from dkist_processing_common.models.flower_pot import ListStem
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
-from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.tags import EXP_TIME_ROUND_DIGITS
 from dkist_processing_common.models.tags import StemName
 from dkist_processing_common.models.task_name import TaskName
@@ -35,7 +35,7 @@ class ObsIpStartTimeBud(TaskUniqueBud):
         )
 
 
-class TaskDatetimeBudBase(Stem):
+class TaskDatetimeBudBase(ListStem):
     """
     Base class for making datetime-related buds.
 
@@ -59,8 +59,6 @@ class TaskDatetimeBudBase(Stem):
         The function used to convert a header into an IP task type
     """
 
-    key_to_petal_dict: dict[str, float]
-
     def __init__(
         self,
         stem_name: str,
@@ -101,20 +99,15 @@ class TaskDatetimeBudBase(Stem):
 
         return SpilledDirt
 
-    def getter(self, key: Hashable) -> tuple[float, ...]:
+    def getter(self) -> tuple[float, ...]:
         """
         Return a tuple of sorted times in unix seconds.
 
-        Parameters
-        ----------
-        key
-            The input key
-
         Returns
         -------
         A tuple that is sorted times in unix seconds
         """
-        return tuple(sorted(list(self.key_to_petal_dict.values())))
+        return tuple(sorted(self.value_list))
 
 
 class CadenceBudBase(TaskDatetimeBudBase):
@@ -134,20 +127,15 @@ class AverageCadenceBud(CadenceBudBase):
     def __init__(self):
         super().__init__(constant_name=BudName.average_cadence)
 
-    def getter(self, key) -> np.float64:
+    def getter(self) -> np.float64:
         """
         Return the mean cadence between frames.
 
-        Parameters
-        ----------
-        key
-            The input key
-
         Returns
         -------
         The mean value of the cadences of the input frames
         """
-        return np.mean(np.diff(super().getter(key)))
+        return np.mean(np.diff(super().getter()))
 
 
 class MaximumCadenceBud(CadenceBudBase):
@@ -156,20 +144,15 @@ class MaximumCadenceBud(CadenceBudBase):
     def __init__(self):
         super().__init__(constant_name=BudName.maximum_cadence)
 
-    def getter(self, key) -> np.float64:
+    def getter(self) -> np.float64:
         """
         Return the maximum cadence between frames.
 
-        Parameters
-        ----------
-        key
-            The input key
-
         Returns
         -------
         The maximum cadence between frames
         """
-        return np.max(np.diff(super().getter(key)))
+        return np.max(np.diff(super().getter()))
 
 
 class MinimumCadenceBud(CadenceBudBase):
@@ -178,20 +161,15 @@ class MinimumCadenceBud(CadenceBudBase):
     def __init__(self):
         super().__init__(constant_name=BudName.minimum_cadence)
 
-    def getter(self, key) -> np.float64:
+    def getter(self) -> np.float64:
         """
         Return the minimum cadence between frames.
 
-        Parameters
-        ----------
-        key
-            The input key
-
         Returns
         -------
         The minimum cadence between frames
         """
-        return np.min(np.diff(super().getter(key)))
+        return np.min(np.diff(super().getter()))
 
 
 class VarianceCadenceBud(CadenceBudBase):
@@ -200,19 +178,15 @@ class VarianceCadenceBud(CadenceBudBase):
     def __init__(self):
         super().__init__(constant_name=BudName.variance_cadence)
 
-    def getter(self, key) -> np.float64:
+    def getter(self) -> np.float64:
         """
         Return the cadence variance between frames.
 
-        Parameters
-        ----------
-        key
-            The input key
         Returns
         -------
         Return the variance of the cadences over the input frames
         """
-        return np.var(np.diff(super().getter(key)))
+        return np.var(np.diff(super().getter()))
 
 
 class TaskDateBeginBud(TaskDatetimeBudBase):
@@ -231,19 +205,16 @@ class TaskDateBeginBud(TaskDatetimeBudBase):
             task_type_parsing_function=task_type_parsing_function,
         )
 
-    def getter(self, key) -> str:
+    def getter(self) -> str:
         """
         Return the earliest date begin for the ip task type converted from unix seconds to datetime string.
 
-        Parameters
-        ----------
-        key
-            The input key
         Returns
         -------
         Return the minimum date begin as a datetime string
         """
-        min_time = super().getter(key)[0]
+        # super().getter() returns a sorted list
+        min_time = super().getter()[0]
         min_time_dt = datetime.fromtimestamp(min_time, tz=timezone.utc)
         return min_time_dt.strftime("%Y-%m-%dT%H:%M:%S.%f")
 
@@ -286,7 +257,7 @@ class ReadoutExpTimeFlower(RoundTimeFlowerBase):
         )
 
 
-class TaskRoundTimeBudBase(Stem):
+class TaskRoundTimeBudBase(SetStem):
     """
     Base class for making buds that need a set of rounded times for computing for specific task types.
 
@@ -310,8 +281,6 @@ class TaskRoundTimeBudBase(Stem):
         The function used to convert a header into an IP task type
     """
 
-    key_to_petal_dict: dict[str, float]
-
     def __init__(
         self,
         stem_name: str,
@@ -349,20 +318,15 @@ class TaskRoundTimeBudBase(Stem):
 
         return SpilledDirt
 
-    def getter(self, key: Hashable) -> tuple[float, ...]:
+    def getter(self) -> tuple[float, ...]:
         """
         Return a tuple of the sorted unique values found.
 
-        Parameters
-        ----------
-        key
-            The input key
-
         Returns
         -------
         A tuple that is the sorted set of unique times
         """
-        return tuple(sorted(set(self.key_to_petal_dict.values())))
+        return tuple(sorted(self.value_set))
 
 
 class TaskExposureTimesBud(TaskRoundTimeBudBase):

dkist_processing_common/parsers/unique_bud.py

@@ -3,15 +3,15 @@
 from enum import StrEnum
 from typing import Callable
 
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
-from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 
 
-class UniqueBud(Stem):
+class UniqueBud(SetStem):
     """
-    Pre-made flower that reads a single header key from all files and raises a ValueError if it is not unique.
+    Pre-made `SetStem` that reads a single header key from all files and raises a ValueError if it is not unique.
 
     Parameters
     ----------
@@ -46,24 +46,17 @@ class UniqueBud(Stem):
         """
         return getattr(fits_obj, self.metadata_key)
 
-    def getter(self, key):
+    def getter(self):
         """
         Get the value for this key and raise an error if it is not unique.
 
-        Parameters
-        ----------
-        key
-            The input key
         Returns
         -------
         The value associated with this input key
         """
-        value_set = set(self.key_to_petal_dict.values())
-        if len(value_set) > 1:
-            raise ValueError(
-                f"Multiple {self.stem_name} values found for key {key}. Values: {value_set}"
-            )
-        return value_set.pop()
+        if len(self.value_set) > 1:
+            raise ValueError(f"Multiple {self.stem_name} values found! Values: {self.value_set}")
+        return self.value_set.pop()
 
 
 class TaskUniqueBud(UniqueBud):

dkist_processing_common/tasks/l1_output_data.py

@@ -2,18 +2,14 @@
 
 import logging
 from abc import ABC
-from itertools import chain
 from pathlib import Path
 from typing import Iterable
 
-from dkist_processing_common.codecs.quality import quality_data_decoder
 from dkist_processing_common.codecs.quality import quality_data_encoder
 from dkist_processing_common.models.message import CatalogFrameMessage
 from dkist_processing_common.models.message import CatalogFrameMessageBody
 from dkist_processing_common.models.message import CatalogObjectMessage
 from dkist_processing_common.models.message import CatalogObjectMessageBody
-from dkist_processing_common.models.message import CreateQualityReportMessage
-from dkist_processing_common.models.message import CreateQualityReportMessageBody
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.mixin.globus import GlobusMixin
 from dkist_processing_common.tasks.mixin.interservice_bus import InterserviceBusMixin
@@ -62,15 +58,19 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
         with self.telemetry_span("Upload quality data"):
             self.transfer_quality_data()
 
-        with self.telemetry_span("Upload science frames"):
+        with self.telemetry_span("Upload output frames"):
             self.transfer_output_frames()
 
     def transfer_output_frames(self):
-        """Create a Globus transfer for all output data."""
-        transfer_items = self.build_output_frame_transfer_list()
+        """Create a Globus transfer for all output data, as well as any available dataset extras."""
+        output_transfer_items = self.build_output_frame_transfer_list()
+        dataset_extra_transfer_items = self.build_dataset_extra_transfer_list()
+        transfer_items = output_transfer_items + dataset_extra_transfer_items
 
         logger.info(
             f"Preparing globus transfer {len(transfer_items)} items: "
+            f"{len(output_transfer_items)} output frames. "
+            f"{len(dataset_extra_transfer_items)} dataset extras. "
             f"recipe_run_id={self.recipe_run_id}. "
             f"transfer_items={transfer_items[:3]}..."
         )
@@ -189,7 +189,9 @@ class SubmitDatasetMetadata(L1OutputDataBase):
 class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
     """Task class for publishing Catalog and Quality Messages."""
 
-    def frame_messages(self, paths: Iterable[Path]) -> list[CatalogFrameMessage]:
+    def frame_messages(
+        self, paths: Iterable[Path], folder_modifier: str | None = None
+    ) -> list[CatalogFrameMessage]:
         """
         Create the frame messages.
 
@@ -197,6 +199,8 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         ----------
         paths
             The input paths for which to publish frame messages
+        folder_modifier
+            A subdirectory to use if the files in paths are not in the base directory
 
         Returns
         -------
@@ -204,7 +208,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         """
         message_bodies = [
             CatalogFrameMessageBody(
-                objectName=self.format_object_key(path=p),
+                objectName=self.format_object_key(path=p, folder_modifier=folder_modifier),
                 conversationId=str(self.recipe_run_id),
                 bucket=self.destination_bucket,
             )
@@ -233,7 +237,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         message_bodies = [
             CatalogObjectMessageBody(
                 objectType=object_type,
-                objectName=self.format_object_key(p),
+                objectName=self.format_object_key(path=p),
                 bucket=self.destination_bucket,
                 conversationId=str(self.recipe_run_id),
                 groupId=self.constants.dataset_id,
@@ -246,19 +250,24 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
     def run(self) -> None:
         """Run method for this task."""
         with self.telemetry_span("Gather output data"):
-            frames = self.read(tags=self.output_frame_tags)
-            movies = self.read(tags=[Tag.output(), Tag.movie()])
+            frames = self.read(
+                tags=self.output_frame_tags
+            )  # frames is kept as a generator as it is much longer than the other file categories
+            extras = list(self.read(tags=self.extra_frame_tags))
+            movies = list(self.read(tags=[Tag.output(), Tag.movie()]))
             quality_data = self.read(tags=[Tag.output(), Tag.quality_data()])
         with self.telemetry_span("Create message objects"):
             messages = []
             messages += self.frame_messages(paths=frames)
             frame_message_count = len(messages)
+            messages += self.frame_messages(paths=extras, folder_modifier="extra")
+            extra_message_count = len(extras)
             messages += self.object_messages(paths=movies, object_type="MOVIE")
-            object_message_count = len(messages) - frame_message_count
+            object_message_count = len(movies)
             dataset_has_quality_data = self.dataset_has_quality_data
             if dataset_has_quality_data:
                 messages += self.object_messages(paths=quality_data, object_type="QDATA")
         with self.telemetry_span(
-            f"Publish messages: {frame_message_count = }, {object_message_count = }, {dataset_has_quality_data = }"
+            f"Publish messages: {frame_message_count = }, {extra_message_count = }, {object_message_count = }, {dataset_has_quality_data = }"
         ):
             self.interservice_bus_publish(messages=messages)

dkist_processing_common/tasks/output_data_base.py

@@ -22,19 +22,23 @@ class OutputDataBase(WorkflowTaskBase, ABC):
         """Get the destination bucket."""
         return self.metadata_store_recipe_run.configuration.destination_bucket
 
-    def format_object_key(self, path: Path) -> str:
+    def format_object_key(self, path: Path, folder_modifier: str | None = None) -> str:
         """
         Convert output paths into object store keys.
 
         Parameters
         ----------
         path: the Path to convert
+        folder_modifier: optional folder name to insert into the path
 
         Returns
         -------
         formatted path in the object store
         """
-        object_key = self.destination_folder / Path(path.name)
+        if folder_modifier:
+            object_key = self.destination_folder / Path(folder_modifier) / Path(path.name)
+        else:
+            object_key = self.destination_folder / Path(path.name)
         return str(object_key)
 
     @property
@@ -52,6 +56,11 @@ class OutputDataBase(WorkflowTaskBase, ABC):
         """Tags that uniquely identify L1 fits frames i.e. the dataset-inventory-able frames."""
         return [Tag.output(), Tag.frame()]
 
+    @property
+    def extra_frame_tags(self) -> list[str]:
+        """Tags that uniquely identify dataset extra fits frames."""
+        return [Tag.output(), Tag.extra()]
+
 
 class TransferDataBase(OutputDataBase, ObjectStoreMixin, ABC):
     """Base class for transferring data from scratch to somewhere else."""
@@ -73,9 +82,21 @@ class TransferDataBase(OutputDataBase, ObjectStoreMixin, ABC):
         """Build a list of GlobusTransfer items corresponding to all OUTPUT (i.e., L1) frames."""
         science_frame_paths: list[Path] = list(self.read(tags=self.output_frame_tags))
 
+        return self.build_transfer_list(science_frame_paths)
+
+    def build_dataset_extra_transfer_list(self) -> list[GlobusTransferItem]:
+        """Build a list of GlobusTransfer items corresponding to all extra dataset files."""
+        extra_paths: list[Path] = list(self.read(tags=self.extra_frame_tags))
+
+        return self.build_transfer_list(paths=extra_paths, destination_folder_modifier="extra")
+
+    def build_transfer_list(
+        self, paths: list[Path], destination_folder_modifier: str | None = None
+    ) -> list[GlobusTransferItem]:
+        """Given a list of paths, build a list of GlobusTransfer items."""
         transfer_items = []
-        for p in science_frame_paths:
-            object_key = self.format_object_key(p)
+        for p in paths:
+            object_key = self.format_object_key(path=p, folder_modifier=destination_folder_modifier)
             destination_path = Path(self.destination_bucket, object_key)
             item = GlobusTransferItem(
                 source_path=p,

dkist_processing_common/tasks/parse_l0_input_data.py

@@ -397,6 +397,8 @@ class ParseDataBase(WorkflowTaskBase, ABC):
         return self.read(
             tags=self.tags_for_input_frames,
             decoder=fits_access_decoder,
+            checksum=False,
+            disable_image_compression=True,
             fits_access_class=self.fits_parsing_class,
         )
 
@@ -419,8 +421,8 @@ class ParseDataBase(WorkflowTaskBase, ABC):
         """
         for stem in constant_pot:
             with self.telemetry_span(f"Setting value of constant {stem.stem_name}"):
-                if len(stem.petals) == 0:
-                    # There are no petals so nothing to do
+                if not stem.can_be_picked:
+                    # Nothing to do
                     continue
                 if stem.bud.value is Thorn:
                     # Must've been a picky bud that passed. We don't want to pick it because it has no value

dkist_processing_common/tasks/transfer_input_data.py

@@ -78,6 +78,7 @@ class TransferL0Data(WorkflowTaskBase, GlobusMixin):
                 self.tag(obj_path, tags=obj.tag)
             else:
                 self.tag(obj_path, tags=[Tag.input(), Tag.frame()])
+        logger.info(f"Tagged {len(input_dataset_objects)} input dataset objects in scratch")
 
     def run(self) -> None:
         """Execute the data transfer."""