dkist-processing-common 12.1.0rc1__py3-none-any.whl → 12.2.0rc1__py3-none-any.whl

changelog/272.feature.rst

@@ -0,0 +1 @@
+Add the framework for using Dataset Extras, that is, other data that can be included with the L1 FITS files generated by the regular pipeline. A new abstract class, WriteL1DatasetExtras, provides helper functionality for use in the instrument pipelines.

dkist_processing_common/models/constants.py

@@ -285,18 +285,24 @@ class ConstantsBase:
     def dark_observing_program_execution_ids(self) -> list[str]:
         """Return the observing program execution ids constant for the dark task."""
         observing_programs = self._db_dict[BudName.dark_observing_program_execution_ids]
+        if isinstance(observing_programs, str):
+            observing_programs = [observing_programs]
         return list(observing_programs)
 
     @property
     def solar_gain_observing_program_execution_ids(self) -> list[str]:
         """Return the observing program execution ids constant for the solar_gain task."""
         observing_programs = self._db_dict[BudName.solar_gain_observing_program_execution_ids]
+        if isinstance(observing_programs, str):
+            observing_programs = [observing_programs]
         return list(observing_programs)
 
     @property
     def polcal_observing_program_execution_ids(self) -> list[str]:
         """Return the observing program execution ids constant."""
         observing_programs = self._db_dict[BudName.polcal_observing_program_execution_ids]
+        if isinstance(observing_programs, str):
+            observing_programs = [observing_programs]
         return list(observing_programs)
 
     @property

dkist_processing_common/models/extras.py

@@ -0,0 +1,35 @@
+"""Autocomplete access to dataset extra header sections."""
+
+from enum import StrEnum
+
+
+class DatasetExtraHeaderSection(StrEnum):
+    """Enum defining the possible header sections for dataset extras."""
+
+    common = "common"
+    aggregate = "aggregate"
+    iptask = "iptask"
+    gos = "gos"
+    wavecal = "wavecal"
+    atlas = "atlas"
+    test = "test"
+
+
+class DatasetExtraType(StrEnum):
+    """Enum defining options for dataset extra names."""
+
+    dark = "DARK"
+    background_light = "BACKGROUND LIGHT"
+    solar_gain = "SOLAR GAIN"
+    characteristic_spectra = "CHARACTERISTIC SPECTRA"
+    modulation_state_offsets = "MODULATION STATE OFFSETS"
+    beam_angles = "BEAM ANGLES"
+    spectral_curvature_shifts = "SPECTRAL CURVATURE SHIFTS"
+    wavelength_calibration_input_spectrum = "WAVELENGTH CALIBRATION INPUT SPECTRUM"
+    wavelength_calibration_reference_spectrum = "WAVELENGTH CALIBRATION REFERENCE SPECTRUM"
+    reference_wavelength_vector = "REFERENCE WAVELENGTH VECTOR"
+    demodulation_matrices = "DEMODULATION MATRICES"
+    polcal_as_science = "POLCAL AS SCIENCE"
+    bad_pixel_map = "BAD PIXEL MAP"
+    beam_offsets = "BEAM OFFSETS"
+    spectral_curvature_scales = "SPECTRAL CURVATURE SCALES"

dkist_processing_common/models/tags.py

@@ -38,6 +38,8 @@ class StemName(StrEnum):
     dataset_inventory = "DATASET_INVENTORY"
     asdf = "ASDF"
     quality_report = "QUALITY_REPORT"
+    # Dataset extras
+    extra = "EXTRA"
 
 
 class Tag:
@@ -450,3 +452,14 @@ class Tag:
         An asdf tag
         """
         return cls.format_tag(StemName.asdf)
+
+    @classmethod
+    def extra(cls) -> str:
+        """
+        Return a dataset extra tag.
+
+        Returns
+        -------
+        A dataset extra tag
+        """
+        return cls.format_tag(StemName.extra)

dkist_processing_common/parsers/id_bud.py

@@ -1,16 +1,17 @@
 """Base classes for ID bud parsing."""
 
+from collections import Counter
 from enum import StrEnum
 from typing import Callable
 from typing import Type
 
-from dkist_processing_common.models.flower_pot import SetStem
+from dkist_processing_common.models.flower_pot import ListStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 
 
-class ContributingIdsBud(SetStem):
+class ContributingIdsBud(ListStem):
     """Base class for contributing ID buds."""
 
     def __init__(self, constant_name: str, metadata_key: str | StrEnum):
@@ -35,13 +36,15 @@ class ContributingIdsBud(SetStem):
 
     def getter(self) -> tuple[str, ...]:
         """
-        Get all ids seen for any type of frame.
+        Get all ids seen for any type of frame, sorted by the number of appearances of that ID.
 
         Returns
         -------
         IDs from all types of frames
         """
-        return tuple(self.value_set)
+        counts = Counter(self.value_list)  # Count the number of appearances of each ID
+        sorted_ids = tuple(str(item) for item, count in counts.most_common())
+        return sorted_ids
 
 
 class TaskContributingIdsBud(ContributingIdsBud):

dkist_processing_common/tasks/l1_output_data.py

@@ -62,15 +62,19 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
         with self.telemetry_span("Upload quality data"):
             self.transfer_quality_data()
 
-        with self.telemetry_span("Upload science frames"):
+        with self.telemetry_span("Upload output frames"):
             self.transfer_output_frames()
 
     def transfer_output_frames(self):
-        """Create a Globus transfer for all output data."""
-        transfer_items = self.build_output_frame_transfer_list()
+        """Create a Globus transfer for all output data, as well as any available dataset extras."""
+        output_transfer_items = self.build_output_frame_transfer_list()
+        dataset_extra_transfer_items = self.build_dataset_extra_transfer_list()
+        transfer_items = output_transfer_items + dataset_extra_transfer_items
 
         logger.info(
             f"Preparing globus transfer {len(transfer_items)} items: "
+            f"{len(output_transfer_items)} output frames. "
+            f"{len(dataset_extra_transfer_items)} dataset extras. "
             f"recipe_run_id={self.recipe_run_id}. "
             f"transfer_items={transfer_items[:3]}..."
         )
@@ -246,19 +250,34 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
     def run(self) -> None:
         """Run method for this task."""
         with self.telemetry_span("Gather output data"):
-            frames = self.read(tags=self.output_frame_tags)
-            movies = self.read(tags=[Tag.output(), Tag.movie()])
+            frames = self.read(
+                tags=self.output_frame_tags
+            )  # frames is kept as a generator as it is much longer than the other file categories
+            extras = list(self.read(tags=self.extra_frame_tags))
+            movies = list(self.read(tags=[Tag.output(), Tag.movie()]))
             quality_data = self.read(tags=[Tag.output(), Tag.quality_data()])
         with self.telemetry_span("Create message objects"):
             messages = []
             messages += self.frame_messages(paths=frames)
             frame_message_count = len(messages)
+            messages += self.frame_messages(paths=extras)
+            extra_message_count = len(extras)
             messages += self.object_messages(paths=movies, object_type="MOVIE")
-            object_message_count = len(messages) - frame_message_count
+            object_message_count = len(movies)
             dataset_has_quality_data = self.dataset_has_quality_data
             if dataset_has_quality_data:
                 messages += self.object_messages(paths=quality_data, object_type="QDATA")
         with self.telemetry_span(
-            f"Publish messages: {frame_message_count = }, {object_message_count = }, {dataset_has_quality_data = }"
+            f"Publish messages: {frame_message_count = }, {extra_message_count = }, {object_message_count = }, {dataset_has_quality_data = }"
         ):
+            messages.append(self.quality_report_message)
+        with self.telemetry_span("Publish Catalog Messages") as publish_span:
+            publish_span.set_attributes(
+                {
+                    "frame_message_count": frame_message_count,
+                    "extra_message_count": extra_message_count,
+                    "object_message_count": object_message_count,
+                    "dataset_has_quality_data": dataset_has_quality_data,
+                }
+            )
             self.interservice_bus_publish(messages=messages)

dkist_processing_common/tasks/output_data_base.py

@@ -22,19 +22,23 @@ class OutputDataBase(WorkflowTaskBase, ABC):
         """Get the destination bucket."""
         return self.metadata_store_recipe_run.configuration.destination_bucket
 
-    def format_object_key(self, path: Path) -> str:
+    def format_object_key(self, path: Path, folder_modifier: str | None = None) -> str:
         """
         Convert output paths into object store keys.
 
         Parameters
         ----------
         path: the Path to convert
+        folder_modifier: optional folder name to insert into the path
 
         Returns
         -------
         formatted path in the object store
         """
-        object_key = self.destination_folder / Path(path.name)
+        if folder_modifier:
+            object_key = self.destination_folder / Path(folder_modifier) / Path(path.name)
+        else:
+            object_key = self.destination_folder / Path(path.name)
         return str(object_key)
 
     @property
@@ -52,6 +56,11 @@ class OutputDataBase(WorkflowTaskBase, ABC):
         """Tags that uniquely identify L1 fits frames i.e. the dataset-inventory-able frames."""
         return [Tag.output(), Tag.frame()]
 
+    @property
+    def extra_frame_tags(self) -> list[str]:
+        """Tags that uniquely identify dataset extra fits frames."""
+        return [Tag.output(), Tag.extra()]
+
 
 class TransferDataBase(OutputDataBase, ObjectStoreMixin, ABC):
     """Base class for transferring data from scratch to somewhere else."""
@@ -73,9 +82,21 @@ class TransferDataBase(OutputDataBase, ObjectStoreMixin, ABC):
         """Build a list of GlobusTransfer items corresponding to all OUTPUT (i.e., L1) frames."""
         science_frame_paths: list[Path] = list(self.read(tags=self.output_frame_tags))
 
+        return self.build_transfer_list(science_frame_paths)
+
+    def build_dataset_extra_transfer_list(self) -> list[GlobusTransferItem]:
+        """Build a list of GlobusTransfer items corresponding to all extra dataset files."""
+        extra_paths: list[Path] = list(self.read(tags=self.extra_frame_tags))
+
+        return self.build_transfer_list(paths=extra_paths, destination_folder_modifier="extra")
+
+    def build_transfer_list(
+        self, paths: list[Path], destination_folder_modifier: str | None = None
+    ) -> list[GlobusTransferItem]:
+        """Given a list of paths, build a list of GlobusTransfer items."""
         transfer_items = []
-        for p in science_frame_paths:
-            object_key = self.format_object_key(p)
+        for p in paths:
+            object_key = self.format_object_key(path=p, folder_modifier=destination_folder_modifier)
             destination_path = Path(self.destination_bucket, object_key)
             item = GlobusTransferItem(
                 source_path=p,

dkist_processing_common/tasks/write_extra.py

@@ -0,0 +1,333 @@
+"""Classes to support the generation of dataset extras."""
+
+import uuid
+from abc import ABC
+from abc import abstractmethod
+from datetime import datetime
+
+import numpy as np
+from astropy.io import fits
+from astropy.time import Time
+from dkist_fits_specifications.utils.formatter import reformat_dataset_extra_header
+from dkist_header_validator.spec_validators import spec_extras_validator
+
+from dkist_processing_common.codecs.fits import fits_hdulist_encoder
+from dkist_processing_common.models.extras import DatasetExtraHeaderSection
+from dkist_processing_common.models.extras import DatasetExtraType
+from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.models.task_name import TaskName
+from dkist_processing_common.tasks.write_l1_base import WriteL1Base
+
+
+class WriteL1DatasetExtras(WriteL1Base, ABC):
+    """Class supporting the construction of dataset extras."""
+
+    def dataset_extra_headers(
+        self,
+        filename: str,
+        task_type: TaskName,
+        extra_name: DatasetExtraType,
+        total_exposure: float | None = None,
+        readout_exposure: float | None = None,
+    ) -> dict:
+        """Provide common FITS header keys for dataset extras."""
+        # Build task specific header values
+        match task_type:
+            case TaskName.dark:
+                task_specific_observing_program_execution_id = (
+                    self.constants.dark_observing_program_execution_ids
+                )
+                task_specific_date_begin = self.constants.dark_date_begin
+                task_specific_date_end = self.constants.dark_date_end
+                task_specific_raw_frames_per_fpa = (
+                    0  # can be updated in construction of dataset extra if required
+                )
+                task_specific_telescope_tracking_mode = (
+                    "None"  # can be updated in construction of dataset extra if required
+                )
+                task_specific_coude_table_tracking_mode = (
+                    "None"  # can be updated in construction of dataset extra if required
+                )
+                task_specific_telescope_scanning_mode = (
+                    "None"  # can be updated in construction of dataset extra if required
+                )
+                task_specific_average_light_level = self.constants.dark_average_light_level
+                task_specific_average_telescope_elevation = (
+                    self.constants.dark_average_telescope_elevation
+                )
+                task_specific_average_coude_table_angle = (
+                    self.constants.dark_average_coude_table_angle
+                )
+                task_specific_average_telescope_azimuth = (
+                    self.constants.dark_average_telescope_azimuth
+                )
+                task_specific_gos_level3_status = self.constants.dark_gos_level3_status
+                task_specific_gos_level3_lamp_status = self.constants.dark_gos_level3_lamp_status
+                task_specific_gos_polarizer_status = self.constants.dark_gos_polarizer_status
+                task_specific_gos_polarizer_angle = self.constants.dark_gos_polarizer_angle
+                task_specific_gos_retarder_status = self.constants.dark_gos_retarder_status
+                task_specific_gos_retarder_angle = self.constants.dark_gos_retarder_angle
+                task_specific_gos_level0_status = self.constants.dark_gos_level0_status
+            case TaskName.solar_gain:
+                task_specific_observing_program_execution_id = (
+                    self.constants.solar_gain_observing_program_execution_ids
+                )
+
+                task_specific_date_begin = self.constants.solar_gain_date_begin
+                task_specific_date_end = self.constants.solar_gain_date_end
+                task_specific_raw_frames_per_fpa = self.constants.solar_gain_num_raw_frames_per_fpa
+                task_specific_telescope_tracking_mode = (
+                    self.constants.solar_gain_telescope_tracking_mode
+                )
+                task_specific_coude_table_tracking_mode = (
+                    self.constants.solar_gain_coude_table_tracking_mode
+                )
+                task_specific_telescope_scanning_mode = (
+                    self.constants.solar_gain_telescope_scanning_mode
+                )
+                task_specific_average_light_level = self.constants.solar_gain_average_light_level
+                task_specific_average_telescope_elevation = (
+                    self.constants.solar_gain_average_telescope_elevation
+                )
+                task_specific_average_coude_table_angle = (
+                    self.constants.solar_gain_average_coude_table_angle
+                )
+                task_specific_average_telescope_azimuth = (
+                    self.constants.solar_gain_average_telescope_azimuth
+                )
+                task_specific_gos_level3_status = self.constants.solar_gain_gos_level3_status
+                task_specific_gos_level3_lamp_status = (
+                    self.constants.solar_gain_gos_level3_lamp_status
+                )
+                task_specific_gos_polarizer_status = self.constants.solar_gain_gos_polarizer_status
+                task_specific_gos_polarizer_angle = self.constants.solar_gain_gos_polarizer_angle
+                task_specific_gos_retarder_status = self.constants.solar_gain_gos_retarder_status
+                task_specific_gos_retarder_angle = self.constants.solar_gain_gos_retarder_angle
+                task_specific_gos_level0_status = self.constants.solar_gain_gos_level0_status
+            case TaskName.polcal:
+                task_specific_observing_program_execution_id = (
+                    self.constants.polcal_observing_program_execution_ids
+                )
+
+                task_specific_date_begin = self.constants.polcal_date_begin
+                task_specific_date_end = self.constants.polcal_date_end
+                task_specific_raw_frames_per_fpa = self.constants.polcal_num_raw_frames_per_fpa
+                task_specific_telescope_tracking_mode = (
+                    self.constants.polcal_telescope_tracking_mode
+                )
+                task_specific_coude_table_tracking_mode = (
+                    self.constants.polcal_coude_table_tracking_mode
+                )
+                task_specific_telescope_scanning_mode = (
+                    self.constants.polcal_telescope_scanning_mode
+                )
+                task_specific_average_light_level = self.constants.polcal_average_light_level
+                task_specific_average_telescope_elevation = (
+                    self.constants.polcal_average_telescope_elevation
+                )
+                task_specific_average_coude_table_angle = (
+                    self.constants.polcal_average_coude_table_angle
+                )
+                task_specific_average_telescope_azimuth = (
+                    self.constants.polcal_average_telescope_azimuth
+                )
+                task_specific_gos_level3_status = None
+                task_specific_gos_level3_lamp_status = None
+                task_specific_gos_polarizer_status = None
+                task_specific_gos_polarizer_angle = None
+                task_specific_gos_retarder_status = None
+                task_specific_gos_retarder_angle = None
+                task_specific_gos_level0_status = None
+            case _:
+                raise ValueError(f"Unsupported task type {task_type}")
+
+        start_datetime = datetime.fromisoformat(task_specific_date_begin)
+        end_datetime = datetime.fromisoformat(task_specific_date_end)
+
+        dataset_extra_header = {
+            DatasetExtraHeaderSection.common: {
+                "BUNIT": "count",
+                "DATE": Time.now().fits,
+                "DATE-BEG": task_specific_date_begin,
+                "DATE-END": task_specific_date_end,
+                "TELAPSE": (end_datetime - start_datetime).total_seconds(),
+                "DATE-AVG": (start_datetime + (end_datetime - start_datetime) / 2).isoformat(),
+                "TIMESYS": "UTC",
+                "ORIGIN": "National Solar Observatory",
+                "TELESCOP": "Daniel K. Inouye Solar Telescope",
+                "OBSRVTRY": "Haleakala High Altitude Observatory Site",
+                "NETWORK": "NSF-DKIST",
+                "INSTRUME": self.constants.instrument,
+                "OBJECT": "unknown",
+                "CAM_ID": self.constants.camera_id,
+                "CAMERA": self.constants.camera_name,
+                "BITDEPTH": self.constants.camera_bit_depth,
+                "XPOSURE": total_exposure,
+                "TEXPOSUR": readout_exposure,
+                "HWBIN1": self.constants.hardware_binning_x,
+                "HWBIN2": self.constants.hardware_binning_y,
+                "SWBIN1": self.constants.software_binning_x,
+                "SWBIN2": self.constants.software_binning_y,
+                "NSUMEXP": task_specific_raw_frames_per_fpa,
+                "DSETID": self.constants.dataset_id,
+                "PROCTYPE": "L1_EXTRA",
+                "RRUNID": self.recipe_run_id,
+                "RECIPEID": self.metadata_store_recipe_run.recipeInstance.recipeId,
+                "RINSTID": self.metadata_store_recipe_run.recipeInstanceId,
+                "FILENAME": filename,
+                "HEAD_URL": "",
+                "INFO_URL": self.docs_base_url,
+                "CAL_URL": "",
+                "CALVERS": self.version_from_module_name(),
+                "IDSPARID": (
+                    parameters.inputDatasetPartId
+                    if (parameters := self.metadata_store_input_dataset_parameters)
+                    else None
+                ),
+                "IDSOBSID": (
+                    observe_frames.inputDatasetPartId
+                    if (observe_frames := self.metadata_store_input_dataset_observe_frames)
+                    else None
+                ),
+                "IDSCALID": (
+                    calibration_frames.inputDatasetPartId
+                    if (calibration_frames := self.metadata_store_input_dataset_calibration_frames)
+                    else None
+                ),
+                "WKFLVERS": self.workflow_version,
+                "WKFLNAME": self.workflow_name,
+                "MANPROCD": self.workflow_had_manual_intervention,
+                "FILE_ID": uuid.uuid4().hex,
+                "OBSPR_ID": task_specific_observing_program_execution_id[
+                    0
+                ],  # The OP IDs are stored sorted by number of appearances of each OP ID in the source task type frames
+                "EXTOBSID": ",".join(task_specific_observing_program_execution_id[1:]),
+                "EXPER_ID": self.constants.experiment_id,
+                "PROP_ID": self.constants.proposal_id,
+                "HLSVERS": self.constants.hls_version,
+                "LINEWAV": self.constants.wavelength,
+                "TELTRACK": (
+                    task_specific_telescope_tracking_mode if task_type != TaskName.dark else None
+                ),
+                "TTBLTRCK": (
+                    task_specific_coude_table_tracking_mode if task_type != TaskName.dark else None
+                ),
+                "TELSCAN": (
+                    task_specific_telescope_scanning_mode if task_type != TaskName.dark else None
+                ),
+                "EXTNAME": extra_name,
+            },
+            DatasetExtraHeaderSection.aggregate: {
+                "AVGLLVL": task_specific_average_light_level,
+                "ATELEVAT": task_specific_average_telescope_elevation,
+                "ATTBLANG": task_specific_average_coude_table_angle,
+                "ATAZIMUT": task_specific_average_telescope_azimuth,
+            },
+            DatasetExtraHeaderSection.iptask: {
+                "IPTASK": "GAIN" if "GAIN" in task_type else task_type,
+            },
+            DatasetExtraHeaderSection.gos: {
+                "LVL3STAT": task_specific_gos_level3_status,
+                "LAMPSTAT": task_specific_gos_level3_lamp_status,
+                "LVL2STAT": task_specific_gos_polarizer_status,
+                "POLANGLE": task_specific_gos_polarizer_angle,
+                "LVL1STAT": task_specific_gos_retarder_status,
+                "RETANGLE": task_specific_gos_retarder_angle,
+                "LVL0STAT": task_specific_gos_level0_status,
+            },
+        }
+
+        # Remove specific headers from dark frames as they don't constants to fill them
+        if task_type == TaskName.dark:
+            for key in ["TELTRACK", "TTBLTRCK", "TELSCAN"]:
+                del dataset_extra_header[DatasetExtraHeaderSection.common][key]
+
+        # Remove specific headers from polcal frames as they don't have constants to fill them
+        if task_type == TaskName.polcal:
+            for key in [
+                "LVL3STAT",
+                "LAMPSTAT",
+                "LVL2STAT",
+                "POLANGLE",
+                "LVL1STAT",
+                "RETANGLE",
+                "LVL0STAT",
+            ]:
+                del dataset_extra_header[DatasetExtraHeaderSection.gos][key]
+
+        return dataset_extra_header
+
+    def build_dataset_extra_header(
+        self,
+        sections: list[DatasetExtraHeaderSection],
+        filename: str,
+        task_type: TaskName,
+        extra_name: DatasetExtraType,
+        total_exposure: float | None = None,
+        readout_exposure: float | None = None,
+    ) -> fits.Header:
+        """Build FITS header for dataset extra file."""
+        header = fits.Header()
+        all_section_headers = self.dataset_extra_headers(
+            filename=filename,
+            task_type=task_type,
+            total_exposure=total_exposure,
+            readout_exposure=readout_exposure,
+            extra_name=extra_name,
+        )
+        for section in sections:
+            header.update(all_section_headers[section].items())
+        return header
+
+    def format_extra_filename(self, extra_name: DatasetExtraType | str, detail: str | None = None):
+        """Format the filename of dataset extras for consistency."""
+        base_filename = f"{self.constants.instrument}_{self.constants.dataset_id}_{extra_name.replace(' ', '-')}"
+        if detail:
+            base_filename += "_" + detail
+        filename_counter = str(self.filename_counter.increment(base_filename))
+        return f"{base_filename}_{filename_counter}.fits"
+
+    def assemble_and_write_dataset_extra(
+        self,
+        data: np.ndarray | list[np.ndarray],
+        header: fits.Header | list[fits.Header],
+        filename: str,
+    ):
+        """Given the data and header information, write the dataset extra."""
+        if isinstance(data, list) and isinstance(header, list):
+            if len(data) != len(header):
+                raise ValueError(
+                    f"{len(data)} data arrays were provided with {len(header)} headers. These must be equal."
+                )
+        if isinstance(data, np.ndarray):
+            data = [data]
+        if isinstance(header, fits.Header):
+            header = [header]
+        hdus = [fits.PrimaryHDU()]  # The first HDU in the list is an empty PrimaryHDU
+        for i, data_array in enumerate(data):
+            tile_size = self.compute_tile_size_for_array(data_array)
+            hdu = fits.CompImageHDU(header=header[i], data=data_array, tile_shape=tile_size)
+            formatted_header = reformat_dataset_extra_header(hdu.header)
+            hdu = fits.CompImageHDU(header=formatted_header, data=hdu.data, tile_shape=tile_size)
+            hdus.append(hdu)
+        self.write(
+            data=fits.HDUList(hdus),
+            tags=[Tag.extra(), Tag.output()],
+            encoder=fits_hdulist_encoder,
+            relative_path=filename,
+        )
+        self.update_framevol(relative_path=filename)
+
+        # Check that the written file passes spec 214 validation if requested
+        if self.validate_l1_on_write:
+            spec_extras_validator.validate(self.scratch.absolute_path(filename), extra=False)
+
+    @abstractmethod
+    def run(self) -> None:
+        """
+        For each dataset extra.
+
+        * Gather the source data in whatever manner is necessary
+        * Build a header using the `build_dataset_extra_header` method to help with header construction
+        * Write the dataset extra using `assemble_and_write_dataset_extra()`
+        """

dkist_processing_common/tasks/write_l1.py

@@ -1,11 +1,9 @@
 """Task(s) for writing level 1 data as 214 compliant fits files."""
 
-import importlib
 import logging
 import uuid
 from abc import ABC
 from abc import abstractmethod
-from functools import cached_property
 from pathlib import Path
 from string import ascii_uppercase
 from typing import Literal
@@ -34,7 +32,7 @@ from dkist_processing_common.models.fried_parameter import r0_valid
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.models.wavelength import WavelengthRange
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
-from dkist_processing_common.tasks import WorkflowTaskBase
+from dkist_processing_common.tasks.write_l1_base import WriteL1Base
 
 logger = logging.getLogger(__name__)
 
@@ -43,7 +41,7 @@ __all__ = ["WriteL1Frame"]
 from dkist_processing_common.tasks.mixin.metadata_store import MetadataStoreMixin
 
 
-class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
+class WriteL1Frame(WriteL1Base, ABC):
     """
     Task to convert final calibrated science frames into spec 214 compliant level 1 frames.
 
@@ -108,45 +106,6 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
                         self.scratch.absolute_path(relative_path), extra=False
                     )
 
-    @cached_property
-    def tile_size_param(self) -> int | None:
-        """Get the tile size parameter for compression."""
-        return self.metadata_store_recipe_run.configuration.tile_size
-
-    @cached_property
-    def validate_l1_on_write(self) -> bool:
-        """Check for validate on write."""
-        return self.metadata_store_recipe_run.configuration.validate_l1_on_write
-
-    @cached_property
-    def workflow_had_manual_intervention(self):
-        """Indicate determining if any provenance capturing steps had manual intervention."""
-        for provenance_record in self.metadata_store_recipe_run.recipeRunProvenances:
-            if provenance_record.isTaskManual:
-                return True
-        return False
-
-    def compute_tile_size_for_array(self, data: np.ndarray) -> list | None:
-        """Determine the tile size to use for compression accounting for array shape minimums."""
-        if self.tile_size_param is None:
-            return None
-        tile_size = []
-        for dim_size in data.shape:
-            if dim_size < self.tile_size_param:
-                tile_size.append(dim_size)
-            else:
-                tile_size.append(self.tile_size_param)
-        return tile_size
-
-    def update_framevol(self, relative_path: str) -> None:
-        """Update FRAMEVOL key to be exactly the size of the file on-disk."""
-        full_path = self.scratch.workflow_base_path / relative_path
-        compressed_size = full_path.stat().st_size / 1024 / 1024
-        hdul = fits.open(full_path, mode="update")
-        hdul[1].header["FRAMEVOL"] = compressed_size
-        hdul.flush()
-        del hdul
-
     def replace_header_values(self, header: fits.Header, data: np.ndarray) -> fits.Header:
         """Replace header values that should already exist with new values."""
         header["FILE_ID"] = uuid.uuid4().hex
@@ -430,18 +389,6 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         )
         return header
 
-    def version_from_module_name(self) -> str:
-        """
-        Get the value of __version__ from a module given its name.
-
-        Returns
-        -------
-        The value of __version__
-        """
-        package = self.__module__.split(".")[0]
-        module = importlib.import_module(package)
-        return module.__version__
-
     @abstractmethod
     def add_dataset_headers(
         self, header: fits.Header, stokes: Literal["I", "Q", "U", "V"]

dkist_processing_common/tasks/write_l1_base.py

@@ -0,0 +1,67 @@
+"""Base class for writing L1 FITS products with headers."""
+
+import importlib
+from abc import ABC
+from functools import cached_property
+
+import numpy as np
+from astropy.io import fits
+
+from dkist_processing_common.tasks import WorkflowTaskBase
+from dkist_processing_common.tasks.mixin.metadata_store import MetadataStoreMixin
+
+
+class WriteL1Base(WorkflowTaskBase, MetadataStoreMixin, ABC):
+    """Base class for writing L1 FITS products with headers."""
+
+    def version_from_module_name(self) -> str:
+        """
+        Get the value of __version__ from a module given its name.
+
+        Returns
+        -------
+        The value of __version__
+        """
+        package = self.__module__.split(".")[0]
+        module = importlib.import_module(package)
+        return module.__version__
+
+    @cached_property
+    def workflow_had_manual_intervention(self) -> bool:
+        """Indicate determining if any provenance capturing steps had manual intervention."""
+        for provenance_record in self.metadata_store_recipe_run.recipeRunProvenances:
+            if provenance_record.isTaskManual:
+                return True
+        return False
+
+    def update_framevol(self, relative_path: str) -> None:
+        """Update FRAMEVOL key to be exactly the size of the file on-disk."""
+        full_path = self.scratch.workflow_base_path / relative_path
+        compressed_size = full_path.stat().st_size / 1024 / 1024
+        hdul = fits.open(full_path, mode="update")
+        for i in range(1, len(hdul)):
+            hdul[i].header["FRAMEVOL"] = compressed_size
+        hdul.flush()
+        del hdul
+
+    @cached_property
+    def tile_size_param(self) -> int | None:
+        """Get the tile size parameter for compression."""
+        return self.metadata_store_recipe_run.configuration.tile_size
+
+    def compute_tile_size_for_array(self, data: np.ndarray) -> list | None:
+        """Determine the tile size to use for compression accounting for array shape minimums."""
+        if self.tile_size_param is None:
+            return None
+        tile_size = []
+        for dim_size in data.shape:
+            if dim_size < self.tile_size_param:
+                tile_size.append(dim_size)
+            else:
+                tile_size.append(self.tile_size_param)
+        return tile_size
+
+    @cached_property
+    def validate_l1_on_write(self) -> bool:
+        """Check for validate on write."""
+        return self.metadata_store_recipe_run.configuration.validate_l1_on_write

dkist_processing_common/tests/test_construct_dataset_extras.py

@@ -0,0 +1,219 @@
+from dataclasses import asdict
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+import pytest
+from astropy.io import fits
+
+from dkist_processing_common._util.scratch import WorkflowFileSystem
+from dkist_processing_common.codecs.fits import fits_array_decoder
+from dkist_processing_common.codecs.fits import fits_array_encoder
+from dkist_processing_common.models.extras import DatasetExtraHeaderSection
+from dkist_processing_common.models.extras import DatasetExtraType
+from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.models.task_name import TaskName
+from dkist_processing_common.tasks.write_extra import WriteL1DatasetExtras
+from dkist_processing_common.tests.mock_metadata_store import RecipeRunResponseMapping
+from dkist_processing_common.tests.mock_metadata_store import fake_gql_client_factory
+from dkist_processing_common.tests.mock_metadata_store import make_default_recipe_run_response
+
+
+@dataclass
+class FakeConstantDb:
+    INSTRUMENT: str = "VBI"
+    DATASET_ID: str = "DATASETID"
+    AVERAGE_CADENCE: float = 10.0
+    MINIMUM_CADENCE: float = 10.0
+    MAXIMUM_CADENCE: float = 10.0
+    VARIANCE_CADENCE: float = 0.0
+    STOKES_PARAMS: tuple = ("I", "Q", "U", "V")
+    PROPOSAL_ID: str = "PROPID1"
+    EXPERIMENT_ID: str = "EXPERID1"
+    CAMERA_ID: str = "CAMERA1"
+    CAMERA_NAME: str = "Camera One"
+    CAMERA_BIT_DEPTH: int = 16
+    HARDWARE_BINNING_X: int = 1
+    HARDWARE_BINNING_Y: int = 1
+    SOFTWARE_BINNING_X: int = 1
+    SOFTWARE_BINNING_Y: int = 1
+    HLS_VERSION: str = "1.8"
+    WAVELENGTH: float = 854.2
+    # Dark
+    DARK_OBSERVING_PROGRAM_EXECUTION_IDS: tuple = ("OP1", "OP2", "OP3")
+    DARK_DATE_BEGIN: str = "2023-01-01T00:00:00"
+    DARK_DATE_END: str = "2023-01-01T01:00:00"
+    DARK_TELESCOPE_TRACKING_MODE: str = "None"
+    DARK_COUDE_TABLE_TRACKING_MODE: str = "fixed coude table angle"
+    DARK_TELESCOPE_SCANNING_MODE: str = "None"
+    DARK_AVERAGE_LIGHT_LEVEL: float = 5.0
+    DARK_AVERAGE_TELESCOPE_ELEVATION: float = 45.0
+    DARK_AVERAGE_COUDE_TABLE_ANGLE: float = 2.0
+    DARK_AVERAGE_TELESCOPE_AZIMUTH: float = 180.0
+    DARK_GOS_LEVEL3_STATUS: str = "clear"
+    DARK_GOS_LEVEL3_LAMP_STATUS: str = "off"
+    DARK_GOS_POLARIZER_STATUS: str = "clear"
+    DARK_GOS_POLARIZER_ANGLE: str = "0.0"
+    DARK_GOS_RETARDER_STATUS: str = "clear"
+    DARK_GOS_RETARDER_ANGLE: str = "0.0"
+    DARK_GOS_LEVEL0_STATUS: str = "DarkShutter"
+    # Solar Gain
+    SOLAR_GAIN_OBSERVING_PROGRAM_EXECUTION_IDS: tuple = ("OP1", "OP2", "OP3")
+    SOLAR_GAIN_DATE_BEGIN: str = "2023-01-01T00:00:00"
+    SOLAR_GAIN_DATE_END: str = "2023-01-01T01:00:00"
+    SOLAR_GAIN_NUM_RAW_FRAMES_PER_FPA: int = 1
+    SOLAR_GAIN_TELESCOPE_TRACKING_MODE: str = "None"
+    SOLAR_GAIN_COUDE_TABLE_TRACKING_MODE: str = "fixed coude table angle"
+    SOLAR_GAIN_TELESCOPE_SCANNING_MODE: str = "None"
+    SOLAR_GAIN_AVERAGE_LIGHT_LEVEL: float = 5.0
+    SOLAR_GAIN_AVERAGE_TELESCOPE_ELEVATION: float = 45.0
+    SOLAR_GAIN_AVERAGE_COUDE_TABLE_ANGLE: float = 2.0
+    SOLAR_GAIN_AVERAGE_TELESCOPE_AZIMUTH: float = 180.0
+    SOLAR_GAIN_GOS_LEVEL3_STATUS: str = "clear"
+    SOLAR_GAIN_GOS_LEVEL3_LAMP_STATUS: str = "off"
+    SOLAR_GAIN_GOS_POLARIZER_STATUS: str = "clear"
+    SOLAR_GAIN_GOS_POLARIZER_ANGLE: str = "0.0"
+    SOLAR_GAIN_GOS_RETARDER_STATUS: str = "clear"
+    SOLAR_GAIN_GOS_RETARDER_ANGLE: str = "0.0"
+    SOLAR_GAIN_GOS_LEVEL0_STATUS: str = "DarkShutter"
+    # Polcal
+    POLCAL_OBSERVING_PROGRAM_EXECUTION_IDS: tuple = ("OP1", "OP2", "OP3")
+    POLCAL_DATE_BEGIN: str = "2023-01-01T00:00:00"
+    POLCAL_DATE_END: str = "2023-01-01T01:00:00"
+    POLCAL_NUM_RAW_FRAMES_PER_FPA: int = 1
+    POLCAL_TELESCOPE_TRACKING_MODE: str = "None"
+    POLCAL_COUDE_TABLE_TRACKING_MODE: str = "fixed coude table angle"
+    POLCAL_TELESCOPE_SCANNING_MODE: str = "None"
+    POLCAL_AVERAGE_LIGHT_LEVEL: float = 5.0
+    POLCAL_AVERAGE_TELESCOPE_ELEVATION: float = 45.0
+    POLCAL_AVERAGE_COUDE_TABLE_ANGLE: float = 2.0
+    POLCAL_AVERAGE_TELESCOPE_AZIMUTH: float = 180.0
+
+
+class ConstructDatasetExtrasTest(WriteL1DatasetExtras):
+    def run(self):
+        # Make a dataset extra for each task type
+
+        for task_type in [
+            TaskName.dark,
+            TaskName.solar_gain,
+        ]:
+            filename = self.format_extra_filename(task_type, detail="BEAM1")
+            data = next(
+                self.read(
+                    tags=[Tag.task(task_type), Tag.intermediate()], decoder=fits_array_decoder
+                )
+            )
+            header = self.build_dataset_extra_header(
+                sections=[
+                    DatasetExtraHeaderSection.common,
+                    DatasetExtraHeaderSection.aggregate,
+                    DatasetExtraHeaderSection.iptask,
+                    DatasetExtraHeaderSection.gos,
+                ],
+                filename=filename,
+                task_type=task_type,
+                total_exposure=0.058,
+                readout_exposure=0.029,
+                extra_name=(
+                    DatasetExtraType.dark if task_type == "DARK" else DatasetExtraType.solar_gain
+                ),
+            )
+
+            self.assemble_and_write_dataset_extra(data=data, header=header, filename=filename)
+
+        task_type = TaskName.polcal
+        filename = self.format_extra_filename(task_type, detail="BEAM1")
+        data = next(
+            self.read(tags=[Tag.task(task_type), Tag.intermediate()], decoder=fits_array_decoder)
+        )
+        header = self.build_dataset_extra_header(
+            sections=[
+                DatasetExtraHeaderSection.common,
+                DatasetExtraHeaderSection.aggregate,
+                DatasetExtraHeaderSection.iptask,
+                DatasetExtraHeaderSection.gos,
+            ],
+            filename=filename,
+            task_type=task_type,
+            total_exposure=0.058,
+            readout_exposure=0.029,
+            extra_name=DatasetExtraType.demodulation_matrices,
+        )
+        self.assemble_and_write_dataset_extra(data=data, header=header, filename=filename)
+
+
+@pytest.fixture()
+def construct_dataset_extras_task(request, recipe_run_id, tmp_path):
+    with ConstructDatasetExtrasTest(
+        recipe_run_id=recipe_run_id,
+        workflow_name="workflow_name",
+        workflow_version="workflow_version",
+    ) as task:
+        task.scratch = WorkflowFileSystem(recipe_run_id=recipe_run_id, scratch_base_path=tmp_path)
+        # Write an intermediate product to be used as the source for each dataset extra
+        for task_type in [
+            TaskName.dark,
+            TaskName.solar_gain,
+            TaskName.polcal,
+        ]:
+            task.write(
+                data=np.random.random(size=(1, 128, 128)),
+                tags=[Tag.task(task_type), Tag.intermediate()],
+                encoder=fits_array_encoder,
+            )
+        task.constants._update(asdict(FakeConstantDb()))
+        yield task
+        task._purge()
+
+
+@pytest.fixture
+def fake_gql_client_default_configuration():
+    """Create GraphQL client Mock that returns result without recipe run configuration."""
+    recipe_run_response = make_default_recipe_run_response()
+    recipe_run_response.configuration = None
+    new_response_mapping = RecipeRunResponseMapping(response=recipe_run_response)
+    FakeGQLClientDefaultConfiguration = fake_gql_client_factory(
+        response_mapping_override=new_response_mapping
+    )
+
+    return FakeGQLClientDefaultConfiguration
+
+
+def test_construct_dataset_extras(
+    construct_dataset_extras_task, mocker, fake_gql_client_default_configuration
+):
+    """
+    Given: A ConstructDatasetExtras task with source data
+    When: Running the ConstructDatasetExtras task
+    Then: A dataset extra files are produced with expected header values
+    """
+    mocker.patch(
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
+        new=fake_gql_client_default_configuration,
+    )
+    task = construct_dataset_extras_task
+    task()
+    dataset_extra_files = list(task.read(tags=[Tag.output(), Tag.extra()]))
+    assert len(dataset_extra_files) == 3
+    for filename in dataset_extra_files:
+        split_filename = Path(filename).name.split("_")
+        assert split_filename[0] == "VBI"
+        assert split_filename[1] == task.constants.dataset_id
+        assert split_filename[-2] == "BEAM1"
+        assert split_filename[-1] == "1.fits"
+        hdul = fits.open(filename)
+        for i in range(1, len(hdul)):
+            assert isinstance(hdul[i], fits.CompImageHDU)
+            header = hdul[i].header
+            assert header["LINEWAV"] == 854.2
+            assert header["INSTRUME"] == "VBI"
+            assert header["ATAZIMUT"] == 180.0
+            assert header["FRAMEVOL"] is not None
+            assert header["IDSOBSID"] == 2
+            assert header["XPOSURE"] == 0.058
+            assert header["OBSPR_ID"] == "OP1"
+            assert header["EXTOBSID"] == "OP2,OP3"
+            assert header["EXTNAME"] in ["DARK", "SOLAR GAIN", "DEMODULATION MATRICES"]
+            assert header["TELAPSE"] == 3600
+            assert header["DATE-AVG"] == "2023-01-01T00:30:00"

dkist_processing_common/tests/test_output_data_base.py

@@ -57,7 +57,11 @@ def transfer_data_task(recipe_run_id, tmp_path, mocker, fake_gql_client):
         unwanted_file_obj = uuid4().hex.encode("utf8")
         task.write(unwanted_file_obj, tags=[Tag.frame()])
 
-        yield task, output_file_obj
+        # Write a dataset extra
+        extra_file_obj = uuid4().hex.encode("utf8")
+        task.write(extra_file_obj, tags=[Tag.output(), Tag.extra()])
+
+        yield task, output_file_obj, extra_file_obj
         task._purge()
 
 
@@ -81,7 +85,7 @@ def test_build_output_frame_transfer_list(transfer_data_task):
     When: Building a transfer list of all OUTPUT frames
     Then: All OUTPUT frames are listed and no non-OUTPUT frames are listed
     """
-    task, output_file_obj = transfer_data_task
+    task, output_file_obj, _ = transfer_data_task
 
     transfer_list = task.build_output_frame_transfer_list()
 
@@ -89,3 +93,21 @@ def test_build_output_frame_transfer_list(transfer_data_task):
     transfer_item = transfer_list[0]
     with transfer_item.source_path.open(mode="rb") as f:
         assert output_file_obj == f.read()
+
+
+def test_build_dataset_extra_transfer_list(transfer_data_task):
+    """
+    Given: A task based on TransferDataBase with some files, some of which are EXTRA_OUTPUT
+    When: Building a transfer list of all EXTRA_OUTPUT frames
+    Then: All EXTRA_OUTPUT frames are listed and no non-EXTRA_OUTPUT frames are listed
+    """
+    task, _, extra_file_obj = transfer_data_task
+
+    transfer_list = task.build_dataset_extra_transfer_list()
+
+    assert len(transfer_list) == 1
+    transfer_item = transfer_list[0]
+    assert "/extra/" not in str(transfer_item.source_path)
+    assert "/extra/" in str(transfer_item.destination_path)
+    with transfer_item.source_path.open(mode="rb") as f:
+        assert extra_file_obj == f.read()

dkist_processing_common/tests/test_transfer_l1_output_data.py

@@ -25,6 +25,7 @@ def transfer_l1_data_task(recipe_run_id, tmp_path, fake_constants_db):
         task.constants._update(fake_constants_db)
         frame_path = task.scratch.workflow_base_path / Path("frame.fits")
         movie_path = task.scratch.workflow_base_path / Path("movie.mp4")
+        extra_path = task.scratch.workflow_base_path / Path("extra.fits")
         with open(frame_path, "w") as f:
             f.write("Frame")
         task.tag(path=frame_path, tags=[Tag.frame(), Tag.output()])

dkist_processing_common/tests/test_trial_catalog.py

@@ -57,6 +57,9 @@ def scratch_with_l1_frames(recipe_run_id, tmp_path) -> WorkflowFileSystem:
         scratch.write(
             file_obj, tags=[Tag.output(), Tag.frame()], relative_path=f"{uuid4().hex}.dat"
         )
+        scratch.write(
+            file_obj, tags=[Tag.output(), Tag.extra()], relative_path=f"{uuid4().hex}.dat"
+        )
 
     return scratch
 

dkist_processing_common/tests/test_write_l1.py

@@ -6,7 +6,6 @@ from unittest.mock import Mock
 import astropy.units as u
 import numpy as np
 import pytest
-from astropy.coordinates import EarthLocation
 from astropy.io import fits
 from astropy.time import Time
 from astropy.time import TimeDelta

{dkist_processing_common-12.1.0rc1.dist-info → dkist_processing_common-12.2.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 12.1.0rc1
+Version: 12.2.0rc1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -16,7 +16,7 @@ Description-Content-Type: text/x-rst
 Requires-Dist: asdf<4.0.0,>=3.5.0
 Requires-Dist: astropy>=7.0.0
 Requires-Dist: dkist-fits-specifications<5.0,>=4.0.0
-Requires-Dist: dkist-header-validator<6.0,>=5.0.0
+Requires-Dist: dkist-header-validator<6.0,>=5.3.0
 Requires-Dist: dkist-processing-core==7.0.1
 Requires-Dist: dkist-processing-pac<4.0,>=3.1
 Requires-Dist: dkist-service-configuration<5.0,>=4.2.0

{dkist_processing_common-12.1.0rc1.dist-info → dkist_processing_common-12.2.0rc1.dist-info}/RECORD

@@ -1,12 +1,5 @@
 changelog/.gitempty,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-changelog/280.misc.rst,sha256=6k9RYfsyA5YMk2-FNPn1ou0FCqacpBfYA_2uGr6D9I8,107
-changelog/282.feature.2.rst,sha256=6OfMMb7pzbStUmHA4YldfprALhPR5Uc_qZrKh9VdenE,176
-changelog/282.feature.rst,sha256=WHCYqxTGJlTrrjttCqRnZizIx2CEz34mLYisW36zJRo,201
-changelog/284.feature.rst,sha256=Um3KupQuiTtX9tpLE4zLk3TW_wc3ZoCGxmFA-dGQJdU,120
-changelog/285.feature.rst,sha256=FdvhAXvf1je9xLNSgZMl32TYomFibMKd1k6cf8ihkXI,142
-changelog/285.misc.rst,sha256=7ie7VKD5yTMjbXs1V-jO2I2_vsCJ49WSK2g-B37NWFs,180
-changelog/286.feature.rst,sha256=Dzqo-mB_plyAhUZGuOJ_e0fXNoRLmwgfXrjfi1hqGHg,189
-changelog/287.misc.rst,sha256=mqc5FFqdtMFrZETadk6m_wkgtoH8kvJjnR-IGMYXDBY,52
+changelog/272.feature.rst,sha256=hqbgG7F741E8KO0k_l9WNFNNi4A5ZYpdABnr34yhbmk,254
 dkist_processing_common/__init__.py,sha256=GQ9EBnYhkOnt-qODclAoLS_g5YVhurxfg1tjVtI9rDI,320
 dkist_processing_common/config.py,sha256=f511KVpK24sQO4dDr4L6PMj5dz0jmWgnx2Y-3DpV0cw,5991
 dkist_processing_common/manual.py,sha256=bIVVyLsbXMh-g_2L3kGROL-1TtJe0_XviHsp7Br31x8,7023
@@ -29,8 +22,9 @@ dkist_processing_common/codecs/str.py,sha256=Xqt5k8IhLc95KiiNiFwB1JWcVVc6T8AfcLr
 dkist_processing_common/fonts/Lato-Regular.ttf,sha256=1jbkaDIx-THtoiLViOlE0IK_0726AvkovuRhwPGFslE,656568
 dkist_processing_common/fonts/__init__.py,sha256=hBvZRtkoGRPlNDWCK-ZePXdSIlThCcjwBDfYaamVgAw,101
 dkist_processing_common/models/__init__.py,sha256=6LMqemdzVZ87fRrpAsbEnTtWZ02_Gu_oajsUlwGRH_Q,74
-dkist_processing_common/models/constants.py,sha256=-UAJh7fz3BhPX4zaRRbQj1J-YnxVMuuvDWxLFmOll5Q,20184
+dkist_processing_common/models/constants.py,sha256=QHLDdMlYab6g5dEivBjkf7O3mW7aJZPSd1OC_EfuRDM,20490
 dkist_processing_common/models/dkist_location.py,sha256=6Nk0wvv4R8ptlrV7BXon7abq4YLvmTdUmPsDN5G8nWc,971
+dkist_processing_common/models/extras.py,sha256=RI4JWOinYl1rRyA4anNDj5nCSIrvwrix_dOod9bcyHA,1207
 dkist_processing_common/models/fits_access.py,sha256=imKqL4-_g6gTR-IeIjZ6qkMhQX3JujdrKFrTd9gOXnw,5605
 dkist_processing_common/models/flower_pot.py,sha256=bSmnfN1r5ASx_E9GtvFzdT7lciLCpMy80TeSLztakZk,12289
 dkist_processing_common/models/fried_parameter.py,sha256=ro_H2Eo3I88lRf1wJjZfTc_XOjhgLt4whIQR_sjAFbM,1609
@@ -41,7 +35,7 @@ dkist_processing_common/models/message_queue_binding.py,sha256=Y4otwkkePrLRSjlry
 dkist_processing_common/models/metric_code.py,sha256=WSLF9yqcVzk9L9u8WBhgtpUYUWYsG4ZFWFRFtezdUCM,848
 dkist_processing_common/models/parameters.py,sha256=9An3SxUEBI-oYHjICQ_q-IIScTfpvVeAFH7jLzBzzWI,9649
 dkist_processing_common/models/quality.py,sha256=TmDVbvPbfl5CIIs1ioD5guLUoEOFTfiJESvDjLTLl5s,3981
-dkist_processing_common/models/tags.py,sha256=0YqiDrismOSds_3XtFBb2dfv0gjMs6CgRv2dJKsSthI,12082
+dkist_processing_common/models/tags.py,sha256=emvQDsLwzfzqSvH2CwvOd85DRlb5pBCKgOgzjbEKxlY,12335
 dkist_processing_common/models/task_name.py,sha256=uAl7qTK4Xx1nqPAhNAe5nAXqxwPwQzAq58YmoccX6xQ,567
 dkist_processing_common/models/telemetry.py,sha256=XVcLNgHCZsP9L7oYiklyLUoqQtWt_xjEkuf70Kbudz4,839
 dkist_processing_common/models/wavelength.py,sha256=4UhRVoNvCHZitXo5S1oRdewadbmGfmDK6wetMV06POA,967
@@ -50,7 +44,7 @@ dkist_processing_common/parsers/average_bud.py,sha256=mTlrBlo7Pe5WnNmui4wQC24-8Q
 dkist_processing_common/parsers/cs_step.py,sha256=6SCSbCgVhEGxJIIrCBByWCmIad_vuje28jECFZ6qlZI,6454
 dkist_processing_common/parsers/dsps_repeat.py,sha256=Jg6oI9-PtFQbQHbGul6_eiRzBKr0Z2HIGOitG0G5CD4,1642
 dkist_processing_common/parsers/experiment_id_bud.py,sha256=LUehIqB56hmDwARph1itSUsPenFHScfrrRuZmcCi4xA,960
-dkist_processing_common/parsers/id_bud.py,sha256=ShHj7bHiPAmMdUW56WfwPTRrCEnBuLu8v45Aaxocd-M,2242
+dkist_processing_common/parsers/id_bud.py,sha256=QvXTOF9kjekNd_M2ZTQwPhhQwoj2m24dqp2OuCNyHB0,2479
 dkist_processing_common/parsers/l0_fits_access.py,sha256=Ol3eo8yyNbGqbN2whhD2jBNoqhOrlwmH8DbMmCLtNk0,1033
 dkist_processing_common/parsers/l1_fits_access.py,sha256=BWojvcFl_RmkrRWHe1WxDCsPeexervlRFPothuXAyoI,4410
 dkist_processing_common/parsers/lookup_bud.py,sha256=IpDtwCsJRiOanDTpoaav3dZXe55WZpYWTnyxIiEWxvk,4047
@@ -67,15 +61,17 @@ dkist_processing_common/parsers/wavelength.py,sha256=P5C9mG8DAKK3GB3vWNRBI5l7pAW
 dkist_processing_common/tasks/__init__.py,sha256=l23ctjNsKJbHbbqaZBMeOPaOtw0hmITEljI_JJ-CVsU,627
 dkist_processing_common/tasks/assemble_movie.py,sha256=1ixDG-f4ODt0vywqVccG3aodLljVO5OGlvuMO9EEvcU,12767
 dkist_processing_common/tasks/base.py,sha256=itAHCvzcodo-q8_AjpWoRaM86BlcjWDpCIiUP7uwmP0,13236
-dkist_processing_common/tasks/l1_output_data.py,sha256=LxGnOSfG0wRQi5r3OuSQueZYSSUG27GBeCDjmg9HCpo,10380
-dkist_processing_common/tasks/output_data_base.py,sha256=r1Bu3FX5zTVj66GTMWtaV_NdhxjyjSm661Bt2Mxmfi4,3685
+dkist_processing_common/tasks/l1_output_data.py,sha256=GKmsjJ9FItQqY4ChreiGZVD_gZvCTJF6rzkVyDyaFP8,11499
+dkist_processing_common/tasks/output_data_base.py,sha256=lJZ3olayA_nUWbxzozfbIbJFD3j2VpxPMt1CB9LB5IY,4763
 dkist_processing_common/tasks/parse_l0_input_data.py,sha256=xcWmwSKwxEeManltCFrqVG224Vk-BYRE6g32VOK--rI,19241
 dkist_processing_common/tasks/quality_metrics.py,sha256=cvGF6tJ8yAvxOvkeG3tWxYwL885BrFW5X3V7_MSzL-A,12481
 dkist_processing_common/tasks/teardown.py,sha256=rwT9lWINVDF11-az_nx-Z5ykMTX_SJCchobpU6sErgk,2360
 dkist_processing_common/tasks/transfer_input_data.py,sha256=4TJqlDjTc503QFvzSmMert99r9KHDwyd72r1kHAVhQA,5879
 dkist_processing_common/tasks/trial_catalog.py,sha256=Yf-BKNCT_OHwJsxxZP8p2eRW04CcY0tw5_YIe1e9RQY,10535
 dkist_processing_common/tasks/trial_output_data.py,sha256=pUdrNlAzuir4AUdfax5_MOplB-A9NrXErMJmAwtJmLA,6811
-dkist_processing_common/tasks/write_l1.py,sha256=Xy834RTp3F95kLcW4ba5gfHMUocfZd82ZQQKnvQcP2M,23204
+dkist_processing_common/tasks/write_extra.py,sha256=YObiq0xMy5Z1QtpahAKjfELlZ_hcTLtiuD1WKZREeg0,16366
+dkist_processing_common/tasks/write_l1.py,sha256=upsaFN3S0r4MasrzjZ6i0gNF_bvQLf_oyYoRogB7odc,21163
+dkist_processing_common/tasks/write_l1_base.py,sha256=iqejlYb3CSagUyi6U56nmgItzrwcQxLIDwgruxZho3A,2474
 dkist_processing_common/tasks/mixin/__init__.py,sha256=-g-DQbU7m1bclJYuFe3Yh757V-35GIDTbstardKQ7nU,68
 dkist_processing_common/tasks/mixin/globus.py,sha256=ugejtZ_MR5LesQYuXM1uICd_yWDE7cZZr0qnWCh75R8,6732
 dkist_processing_common/tasks/mixin/interservice_bus.py,sha256=M6R922l7gJSmmU_vswUXxy-c5DWNrIRjQu9H9CSgGfU,1081
@@ -92,6 +88,7 @@ dkist_processing_common/tests/test_assemble_quality.py,sha256=-F22jMY6mPy65VZ1TZ
 dkist_processing_common/tests/test_base.py,sha256=gsyBG2R6Ufx7CzbHeGMagUwM9yCfpN4gCSZ6-aH2q48,6643
 dkist_processing_common/tests/test_codecs.py,sha256=WpF15UYklpNRgETI4EwXsgbNzxMcHlelfpprBbupC0I,23907
 dkist_processing_common/tests/test_constants.py,sha256=I_KcJs7ScCn53GYhEO6qjWrrnfZuyC1IVYOy87Pjlg4,6565
+dkist_processing_common/tests/test_construct_dataset_extras.py,sha256=LjFoLt1oChS298T8vmMK3xT0-5LSxRO3gUexxYSVY7w,9113
 dkist_processing_common/tests/test_cs_step.py,sha256=RA0QD3D8eaL3YSOL_gIJ9wkngy14RQ2jbD-05KAziW4,2408
 dkist_processing_common/tests/test_dkist_location.py,sha256=-_OoSw4SZDLFyIuOltHvM6PQjxm5hTiJQsiTGZ8Sadc,456
 dkist_processing_common/tests/test_fits_access.py,sha256=a50B4IAAH5NH5zeudTqyy0b5uWKJwJuzQLUdK1LoOHM,12832
@@ -101,7 +98,7 @@ dkist_processing_common/tests/test_input_dataset.py,sha256=wnQbZxBYywG5CEXces2WW
 dkist_processing_common/tests/test_interservice_bus.py,sha256=QrBeZ8dh497h6nxA8-aVUIGDcSj8y9DIXIk9I_HkXr0,3001
 dkist_processing_common/tests/test_interservice_bus_mixin.py,sha256=IptJkW7Qeu2Y742NKXEgkok2VdS600keLgCD3Y9iw3A,4131
 dkist_processing_common/tests/test_manual_processing.py,sha256=iHF7yQPlar9niYAGXtFv28Gw3Undlds38yMfszk4ccY,1037
-dkist_processing_common/tests/test_output_data_base.py,sha256=D8b1XKvbE3C5cGOiHq58yJ2pzQL3iL0wLZy_AkDdB9Y,3085
+dkist_processing_common/tests/test_output_data_base.py,sha256=VoXW7g5yE2Lzb-HpyhVQYbPpiCZ7YYQAuShoAFVO5nE,3983
 dkist_processing_common/tests/test_parameters.py,sha256=CUEUIGBPMCUXPll0G0UxFDbMXi8lmnjRwXBarGX1PAQ,14033
 dkist_processing_common/tests/test_parse_l0_input_data.py,sha256=7yn1VGwC3S-0JOYIMflNHhaXEnPhwjJtewD2WxBgugM,13239
 dkist_processing_common/tests/test_publish_catalog_messages.py,sha256=7WRsEwoLHGeaCmLTAW4tU_BlZw0e3hwx65uWSGzfuYE,2393
@@ -115,11 +112,11 @@ dkist_processing_common/tests/test_task_name.py,sha256=kqFr59XX2K87xzfTlClzDV4-J
 dkist_processing_common/tests/test_task_parsing.py,sha256=2_OOmeZQWD17XAd_ECYmodJzD_iRIBKjCYdGh38BOx4,4421
 dkist_processing_common/tests/test_teardown.py,sha256=DaliHSGsiQBZaFkf5wb3XBo6rHNPmx2bmQtVymYeBN4,5601
 dkist_processing_common/tests/test_transfer_input_data.py,sha256=eyAAWXpTHQ8aew87-MncWpYBn4DAZrTSOL3LvlQfR5Q,12611
-dkist_processing_common/tests/test_transfer_l1_output_data.py,sha256=PVGDJBEUk4kAeu8ivrhlCE7yd29R18t9kZLFx-mpBwY,2063
-dkist_processing_common/tests/test_trial_catalog.py,sha256=CxjtVABE5Fw2EvyXR56IJ3PPi9QvEOjccH0OzzRWk30,9424
+dkist_processing_common/tests/test_transfer_l1_output_data.py,sha256=OV2XMEkCbEuL9_i2S3P9Jfyf15tcBkAP3JNE8Jn_A9k,2137
+dkist_processing_common/tests/test_trial_catalog.py,sha256=vnGLV859shiYEP6qOVoYga9vCBMRs6UkNnGtnaps26Q,9548
 dkist_processing_common/tests/test_trial_output_data.py,sha256=fu3iGNV_FI8LOacezyt4HvXnxY3g1_UiBuRI63yz5Oo,11977
 dkist_processing_common/tests/test_workflow_task_base.py,sha256=LTVusltNrsGUOvw9G323am4CXebgE4tJhP6gZCcS0CQ,10457
-dkist_processing_common/tests/test_write_l1.py,sha256=alN-lozKEm6vKNdhtvzjnuPqv-NjHyUg16Op7SkMH-c,27964
+dkist_processing_common/tests/test_write_l1.py,sha256=qraecD9Vv6jsqg00XOSlM1IdK9l8i36VoHLoLj8ypiU,27918
 docs/Makefile,sha256=qnlVz6PuBqE39NfHWuUnHhNEA-EFgT2-WJNNNy9ttfk,4598
 docs/changelog.rst,sha256=S2jPASsWlQxSlAPqdvNrYvhk9k3FcFWNXFNDYXBSjl4,120
 docs/conf.py,sha256=7W2iHKs3J5RhAz0JZafC_UnfMvcpZN7j4LLUmQtk2D0,1891
@@ -128,7 +125,7 @@ docs/landing_page.rst,sha256=aPAuXFhBx73lEZ59B6E6JXxkK0LlxzD0n-HXqHrfumQ,746
 docs/make.bat,sha256=mBAhtURwhQ7yc95pqwJzlhqBSvRknr1aqZ5s8NKvdKs,4513
 docs/requirements.txt,sha256=Kbl_X4c7RQZw035YTeNB63We6I7pvXFU4T0Uflp2yDY,29
 licenses/LICENSE.rst,sha256=piZaQplkzOMmH1NXg6QIdo9wwo9pPCoHkvm2-DmH76E,1462
-dkist_processing_common-12.1.0rc1.dist-info/METADATA,sha256=Sv5XyrgF8y0Gx_38hkXIqvcBsPKip3ug5Lfl0mP6lUs,14214
-dkist_processing_common-12.1.0rc1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-dkist_processing_common-12.1.0rc1.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
-dkist_processing_common-12.1.0rc1.dist-info/RECORD,,
+dkist_processing_common-12.2.0rc1.dist-info/METADATA,sha256=jYcIT0L5XRExn309Uky-DKXEKarMZJaHgfu0xF3EXl0,14214
+dkist_processing_common-12.2.0rc1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+dkist_processing_common-12.2.0rc1.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
+dkist_processing_common-12.2.0rc1.dist-info/RECORD,,

changelog/280.misc.rst

@@ -1 +0,0 @@
-Speed up the reading of INPUT files in Parse tasks by turning off image decompression and checksum checks.

changelog/282.feature.2.rst

@@ -1,2 +0,0 @@
-Add `SetStem` base class that has all the benefits of `ListStem` but also gets a speedup by storing values in a `set` for
-cases where repeated values don't need to be tracked.

changelog/282.feature.rst

@@ -1,2 +0,0 @@
-Add `ListStem` base class for huge speedup in cases where the keys don't matter and the `getter` logic only depends on the
-list of values computed by `setter`. This is the case for most (all?) "Buds".

changelog/284.feature.rst

@@ -1 +0,0 @@
-Speed up parsing of the `*CadenceBud`, `TaskDateBeginBud`, and `[Task]NearFloatBud` by basing these buds on `ListStem`.

changelog/285.feature.rst

@@ -1,2 +0,0 @@
-Speed up `NumCSStepBud`, `[Task]UniqueBud`, `[Task]ContributingIdsBud`, and `TaskRoundTimeBudBase` parsing by basing
-these buds on `SetStem`.

changelog/285.misc.rst

@@ -1,2 +0,0 @@
-Update `RetarderNameBud` to drop "clear" values (i.e., the retarder is out of the beam) in the `setter` instead of the `getter`.
-This brings it in line with standard Bud-practice.

changelog/286.feature.rst

@@ -1,2 +0,0 @@
-Speed up `CSStepFlower` parsing by using an internal set to keep track of the unique `CSStep` objects. This removes the
-need to compute the unique set when computing the tag for each file.

changelog/287.misc.rst

@@ -1 +0,0 @@
-Convert the TimeLookupBud to be a SetStem constant.