dkist-processing-common 12.1.0rc1__tar.gz → 12.2.0__tar.gz

{dkist_processing_common-12.1.0rc1 → dkist_processing_common-12.2.0}/CHANGELOG.rst

@@ -1,3 +1,38 @@
+v12.2.0 (2026-01-30)
+====================
+
+Features
+--------
+
+- Add the framework for using Dataset Extras, that is, other data that can be included with the L1 FITS files generated by the regular pipeline. A new abstract class, WriteL1DatasetExtras, provides helper functionality for use in the instrument pipelines. (`#272 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/272>`__)
+
+
+v12.1.0 (2026-01-26)
+====================
+
+Features
+--------
+
+- Add `ListStem` base class for huge speedup in cases where the keys don't matter and the `getter` logic only depends on the
+  list of values computed by `setter`. This is the case for most (all?) "Buds". (`#282 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/282>`__)
+- Add `SetStem` base class that has all the benefits of `ListStem` but also gets a speedup by storing values in a `set` for
+  cases where repeated values don't need to be tracked. (`#282 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/282>`__)
+- Speed up parsing of the `*CadenceBud`, `TaskDateBeginBud`, and `[Task]NearFloatBud` by basing these buds on `ListStem`. (`#284 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/284>`__)
+- Speed up `NumCSStepBud`, `[Task]UniqueBud`, `[Task]ContributingIdsBud`, and `TaskRoundTimeBudBase` parsing by basing
+  these buds on `SetStem`. (`#285 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/285>`__)
+- Speed up `CSStepFlower` parsing by using an internal set to keep track of the unique `CSStep` objects. This removes the
+  need to compute the unique set when computing the tag for each file. (`#286 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/286>`__)
+
+
+Misc
+----
+
+- Speed up the reading of INPUT files in Parse tasks by turning off image decompression and checksum checks. (`#280 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/280>`__)
+- Update `RetarderNameBud` to drop "clear" values (i.e., the retarder is out of the beam) in the `setter` instead of the `getter`.
+  This brings it in line with standard Bud-practice. (`#285 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/285>`__)
+- Convert the TimeLookupBud to be a SetStem constant. (`#287 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/287>`__)
+
+
 v12.0.0 (2026-01-22)
 ====================
 

{dkist_processing_common-12.1.0rc1 → dkist_processing_common-12.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 12.1.0rc1
+Version: 12.2.0
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -16,7 +16,7 @@ Description-Content-Type: text/x-rst
 Requires-Dist: asdf<4.0.0,>=3.5.0
 Requires-Dist: astropy>=7.0.0
 Requires-Dist: dkist-fits-specifications<5.0,>=4.0.0
-Requires-Dist: dkist-header-validator<6.0,>=5.0.0
+Requires-Dist: dkist-header-validator<6.0,>=5.3.0
 Requires-Dist: dkist-processing-core==7.0.1
 Requires-Dist: dkist-processing-pac<4.0,>=3.1
 Requires-Dist: dkist-service-configuration<5.0,>=4.2.0

{dkist_processing_common-12.1.0rc1 → dkist_processing_common-12.2.0}/dkist_processing_common/models/constants.py

@@ -285,18 +285,24 @@ class ConstantsBase:
     def dark_observing_program_execution_ids(self) -> list[str]:
         """Return the observing program execution ids constant for the dark task."""
         observing_programs = self._db_dict[BudName.dark_observing_program_execution_ids]
+        if isinstance(observing_programs, str):
+            observing_programs = [observing_programs]
         return list(observing_programs)
 
     @property
     def solar_gain_observing_program_execution_ids(self) -> list[str]:
         """Return the observing program execution ids constant for the solar_gain task."""
         observing_programs = self._db_dict[BudName.solar_gain_observing_program_execution_ids]
+        if isinstance(observing_programs, str):
+            observing_programs = [observing_programs]
         return list(observing_programs)
 
     @property
     def polcal_observing_program_execution_ids(self) -> list[str]:
         """Return the observing program execution ids constant."""
         observing_programs = self._db_dict[BudName.polcal_observing_program_execution_ids]
+        if isinstance(observing_programs, str):
+            observing_programs = [observing_programs]
         return list(observing_programs)
 
     @property
@@ -467,14 +473,14 @@ class ConstantsBase:
         return self._db_dict[BudName.solar_gain_gos_polarizer_status]
 
     @property
-    def dark_gos_polarizer_angle(self) -> float:
+    def dark_gos_polarizer_angle(self) -> str:
         """Return the gos polarizer angle constant for the dark task."""
-        return self._db_dict[BudName.dark_gos_polarizer_angle]
+        return str(self._db_dict[BudName.dark_gos_polarizer_angle])
 
     @property
-    def solar_gain_gos_polarizer_angle(self) -> float:
+    def solar_gain_gos_polarizer_angle(self) -> str:
         """Return the gos polarizer angle constant for the solar gain task."""
-        return self._db_dict[BudName.solar_gain_gos_polarizer_angle]
+        return str(self._db_dict[BudName.solar_gain_gos_polarizer_angle])
 
     @property
     def dark_gos_retarder_status(self) -> str:
@@ -487,14 +493,14 @@ class ConstantsBase:
         return self._db_dict[BudName.solar_gain_gos_retarder_status]
 
     @property
-    def dark_gos_retarder_angle(self) -> float:
+    def dark_gos_retarder_angle(self) -> str:
         """Return the gos retarder angle constant for the dark task."""
-        return self._db_dict[BudName.dark_gos_retarder_angle]
+        return str(self._db_dict[BudName.dark_gos_retarder_angle])
 
     @property
-    def solar_gain_gos_retarder_angle(self) -> float:
+    def solar_gain_gos_retarder_angle(self) -> str:
         """Return the gos retarder angle constant for the solar gain task."""
-        return self._db_dict[BudName.solar_gain_gos_retarder_angle]
+        return str(self._db_dict[BudName.solar_gain_gos_retarder_angle])
 
     @property
     def dark_gos_level0_status(self) -> str:

dkist_processing_common-12.2.0/dkist_processing_common/models/extras.py

@@ -0,0 +1,35 @@
+"""Autocomplete access to dataset extra header sections."""
+
+from enum import StrEnum
+
+
+class DatasetExtraHeaderSection(StrEnum):
+    """Enum defining the possible header sections for dataset extras."""
+
+    common = "common"
+    aggregate = "aggregate"
+    iptask = "iptask"
+    gos = "gos"
+    wavecal = "wavecal"
+    atlas = "atlas"
+    test = "test"
+
+
+class DatasetExtraType(StrEnum):
+    """Enum defining options for dataset extra names."""
+
+    dark = "DARK"
+    background_light = "BACKGROUND LIGHT"
+    solar_gain = "SOLAR GAIN"
+    characteristic_spectra = "CHARACTERISTIC SPECTRA"
+    modulation_state_offsets = "MODULATION STATE OFFSETS"
+    beam_angles = "BEAM ANGLES"
+    spectral_curvature_shifts = "SPECTRAL CURVATURE SHIFTS"
+    wavelength_calibration_input_spectrum = "WAVELENGTH CALIBRATION INPUT SPECTRUM"
+    wavelength_calibration_reference_spectrum = "WAVELENGTH CALIBRATION REFERENCE SPECTRUM"
+    reference_wavelength_vector = "REFERENCE WAVELENGTH VECTOR"
+    demodulation_matrices = "DEMODULATION MATRICES"
+    polcal_as_science = "POLCAL AS SCIENCE"
+    bad_pixel_map = "BAD PIXEL MAP"
+    beam_offsets = "BEAM OFFSETS"
+    spectral_curvature_scales = "SPECTRAL CURVATURE SCALES"

{dkist_processing_common-12.1.0rc1 → dkist_processing_common-12.2.0}/dkist_processing_common/models/tags.py

@@ -38,6 +38,8 @@ class StemName(StrEnum):
     dataset_inventory = "DATASET_INVENTORY"
     asdf = "ASDF"
     quality_report = "QUALITY_REPORT"
+    # Dataset extras
+    extra = "EXTRA"
 
 
 class Tag:
@@ -450,3 +452,14 @@ class Tag:
         An asdf tag
         """
         return cls.format_tag(StemName.asdf)
+
+    @classmethod
+    def extra(cls) -> str:
+        """
+        Return a dataset extra tag.
+
+        Returns
+        -------
+        A dataset extra tag
+        """
+        return cls.format_tag(StemName.extra)

{dkist_processing_common-12.1.0rc1 → dkist_processing_common-12.2.0}/dkist_processing_common/parsers/id_bud.py

@@ -1,16 +1,17 @@
 """Base classes for ID bud parsing."""
 
+from collections import Counter
 from enum import StrEnum
 from typing import Callable
 from typing import Type
 
-from dkist_processing_common.models.flower_pot import SetStem
+from dkist_processing_common.models.flower_pot import ListStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 
 
-class ContributingIdsBud(SetStem):
+class ContributingIdsBud(ListStem):
     """Base class for contributing ID buds."""
 
     def __init__(self, constant_name: str, metadata_key: str | StrEnum):
@@ -35,13 +36,15 @@ class ContributingIdsBud(SetStem):
 
     def getter(self) -> tuple[str, ...]:
         """
-        Get all ids seen for any type of frame.
+        Get all ids seen for any type of frame, sorted by the number of appearances of that ID.
 
         Returns
         -------
         IDs from all types of frames
         """
-        return tuple(self.value_set)
+        counts = Counter(self.value_list)  # Count the number of appearances of each ID
+        sorted_ids = tuple(str(item) for item, count in counts.most_common())
+        return sorted_ids
 
 
 class TaskContributingIdsBud(ContributingIdsBud):

{dkist_processing_common-12.1.0rc1 → dkist_processing_common-12.2.0}/dkist_processing_common/tasks/l1_output_data.py

@@ -2,18 +2,14 @@
 
 import logging
 from abc import ABC
-from itertools import chain
 from pathlib import Path
 from typing import Iterable
 
-from dkist_processing_common.codecs.quality import quality_data_decoder
 from dkist_processing_common.codecs.quality import quality_data_encoder
 from dkist_processing_common.models.message import CatalogFrameMessage
 from dkist_processing_common.models.message import CatalogFrameMessageBody
 from dkist_processing_common.models.message import CatalogObjectMessage
 from dkist_processing_common.models.message import CatalogObjectMessageBody
-from dkist_processing_common.models.message import CreateQualityReportMessage
-from dkist_processing_common.models.message import CreateQualityReportMessageBody
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.mixin.globus import GlobusMixin
 from dkist_processing_common.tasks.mixin.interservice_bus import InterserviceBusMixin
@@ -62,15 +58,19 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
         with self.telemetry_span("Upload quality data"):
             self.transfer_quality_data()
 
-        with self.telemetry_span("Upload science frames"):
+        with self.telemetry_span("Upload output frames"):
             self.transfer_output_frames()
 
     def transfer_output_frames(self):
-        """Create a Globus transfer for all output data."""
-        transfer_items = self.build_output_frame_transfer_list()
+        """Create a Globus transfer for all output data, as well as any available dataset extras."""
+        output_transfer_items = self.build_output_frame_transfer_list()
+        dataset_extra_transfer_items = self.build_dataset_extra_transfer_list()
+        transfer_items = output_transfer_items + dataset_extra_transfer_items
 
         logger.info(
             f"Preparing globus transfer {len(transfer_items)} items: "
+            f"{len(output_transfer_items)} output frames. "
+            f"{len(dataset_extra_transfer_items)} dataset extras. "
             f"recipe_run_id={self.recipe_run_id}. "
             f"transfer_items={transfer_items[:3]}..."
         )
@@ -189,7 +189,9 @@ class SubmitDatasetMetadata(L1OutputDataBase):
 class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
     """Task class for publishing Catalog and Quality Messages."""
 
-    def frame_messages(self, paths: Iterable[Path]) -> list[CatalogFrameMessage]:
+    def frame_messages(
+        self, paths: Iterable[Path], folder_modifier: str | None = None
+    ) -> list[CatalogFrameMessage]:
         """
         Create the frame messages.
 
@@ -197,6 +199,8 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         ----------
         paths
             The input paths for which to publish frame messages
+        folder_modifier
+            A subdirectory to use if the files in paths are not in the base directory
 
         Returns
         -------
@@ -204,7 +208,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         """
         message_bodies = [
             CatalogFrameMessageBody(
-                objectName=self.format_object_key(path=p),
+                objectName=self.format_object_key(path=p, folder_modifier=folder_modifier),
                 conversationId=str(self.recipe_run_id),
                 bucket=self.destination_bucket,
             )
@@ -233,7 +237,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         message_bodies = [
             CatalogObjectMessageBody(
                 objectType=object_type,
-                objectName=self.format_object_key(p),
+                objectName=self.format_object_key(path=p),
                 bucket=self.destination_bucket,
                 conversationId=str(self.recipe_run_id),
                 groupId=self.constants.dataset_id,
@@ -246,19 +250,24 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
     def run(self) -> None:
         """Run method for this task."""
         with self.telemetry_span("Gather output data"):
-            frames = self.read(tags=self.output_frame_tags)
-            movies = self.read(tags=[Tag.output(), Tag.movie()])
+            frames = self.read(
+                tags=self.output_frame_tags
+            )  # frames is kept as a generator as it is much longer than the other file categories
+            extras = list(self.read(tags=self.extra_frame_tags))
+            movies = list(self.read(tags=[Tag.output(), Tag.movie()]))
             quality_data = self.read(tags=[Tag.output(), Tag.quality_data()])
         with self.telemetry_span("Create message objects"):
             messages = []
             messages += self.frame_messages(paths=frames)
             frame_message_count = len(messages)
+            messages += self.frame_messages(paths=extras, folder_modifier="extra")
+            extra_message_count = len(extras)
             messages += self.object_messages(paths=movies, object_type="MOVIE")
-            object_message_count = len(messages) - frame_message_count
+            object_message_count = len(movies)
             dataset_has_quality_data = self.dataset_has_quality_data
             if dataset_has_quality_data:
                 messages += self.object_messages(paths=quality_data, object_type="QDATA")
         with self.telemetry_span(
-            f"Publish messages: {frame_message_count = }, {object_message_count = }, {dataset_has_quality_data = }"
+            f"Publish messages: {frame_message_count = }, {extra_message_count = }, {object_message_count = }, {dataset_has_quality_data = }"
         ):
             self.interservice_bus_publish(messages=messages)

{dkist_processing_common-12.1.0rc1 → dkist_processing_common-12.2.0}/dkist_processing_common/tasks/output_data_base.py

@@ -22,19 +22,23 @@ class OutputDataBase(WorkflowTaskBase, ABC):
         """Get the destination bucket."""
         return self.metadata_store_recipe_run.configuration.destination_bucket
 
-    def format_object_key(self, path: Path) -> str:
+    def format_object_key(self, path: Path, folder_modifier: str | None = None) -> str:
         """
         Convert output paths into object store keys.
 
         Parameters
         ----------
         path: the Path to convert
+        folder_modifier: optional folder name to insert into the path
 
         Returns
         -------
         formatted path in the object store
         """
-        object_key = self.destination_folder / Path(path.name)
+        if folder_modifier:
+            object_key = self.destination_folder / Path(folder_modifier) / Path(path.name)
+        else:
+            object_key = self.destination_folder / Path(path.name)
         return str(object_key)
 
     @property
@@ -52,6 +56,11 @@ class OutputDataBase(WorkflowTaskBase, ABC):
         """Tags that uniquely identify L1 fits frames i.e. the dataset-inventory-able frames."""
         return [Tag.output(), Tag.frame()]
 
+    @property
+    def extra_frame_tags(self) -> list[str]:
+        """Tags that uniquely identify dataset extra fits frames."""
+        return [Tag.output(), Tag.extra()]
+
 
 class TransferDataBase(OutputDataBase, ObjectStoreMixin, ABC):
     """Base class for transferring data from scratch to somewhere else."""
@@ -73,9 +82,21 @@ class TransferDataBase(OutputDataBase, ObjectStoreMixin, ABC):
         """Build a list of GlobusTransfer items corresponding to all OUTPUT (i.e., L1) frames."""
         science_frame_paths: list[Path] = list(self.read(tags=self.output_frame_tags))
 
+        return self.build_transfer_list(science_frame_paths)
+
+    def build_dataset_extra_transfer_list(self) -> list[GlobusTransferItem]:
+        """Build a list of GlobusTransfer items corresponding to all extra dataset files."""
+        extra_paths: list[Path] = list(self.read(tags=self.extra_frame_tags))
+
+        return self.build_transfer_list(paths=extra_paths, destination_folder_modifier="extra")
+
+    def build_transfer_list(
+        self, paths: list[Path], destination_folder_modifier: str | None = None
+    ) -> list[GlobusTransferItem]:
+        """Given a list of paths, build a list of GlobusTransfer items."""
         transfer_items = []
-        for p in science_frame_paths:
-            object_key = self.format_object_key(p)
+        for p in paths:
+            object_key = self.format_object_key(path=p, folder_modifier=destination_folder_modifier)
             destination_path = Path(self.destination_bucket, object_key)
             item = GlobusTransferItem(
                 source_path=p,