dkist-processing-common 12.1.0rc1__tar.gz → 12.2.0rc2__tar.gz

{dkist_processing_common-12.1.0rc1 → dkist_processing_common-12.2.0rc2}/CHANGELOG.rst

@@ -1,3 +1,29 @@
+v12.1.0 (2026-01-26)
+====================
+
+Features
+--------
+
+- Add `ListStem` base class for huge speedup in cases where the keys don't matter and the `getter` logic only depends on the
+  list of values computed by `setter`. This is the case for most (all?) "Buds". (`#282 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/282>`__)
+- Add `SetStem` base class that has all the benefits of `ListStem` but also gets a speedup by storing values in a `set` for
+  cases where repeated values don't need to be tracked. (`#282 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/282>`__)
+- Speed up parsing of the `*CadenceBud`, `TaskDateBeginBud`, and `[Task]NearFloatBud` by basing these buds on `ListStem`. (`#284 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/284>`__)
+- Speed up `NumCSStepBud`, `[Task]UniqueBud`, `[Task]ContributingIdsBud`, and `TaskRoundTimeBudBase` parsing by basing
+  these buds on `SetStem`. (`#285 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/285>`__)
+- Speed up `CSStepFlower` parsing by using an internal set to keep track of the unique `CSStep` objects. This removes the
+  need to compute the unique set when computing the tag for each file. (`#286 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/286>`__)
+
+
+Misc
+----
+
+- Speed up the reading of INPUT files in Parse tasks by turning off image decompression and checksum checks. (`#280 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/280>`__)
+- Update `RetarderNameBud` to drop "clear" values (i.e., the retarder is out of the beam) in the `setter` instead of the `getter`.
+  This brings it in line with standard Bud-practice. (`#285 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/285>`__)
+- Convert the TimeLookupBud to be a SetStem constant. (`#287 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/287>`__)
+
+
 v12.0.0 (2026-01-22)
 ====================
 

{dkist_processing_common-12.1.0rc1 → dkist_processing_common-12.2.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 12.1.0rc1
+Version: 12.2.0rc2
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -16,7 +16,7 @@ Description-Content-Type: text/x-rst
 Requires-Dist: asdf<4.0.0,>=3.5.0
 Requires-Dist: astropy>=7.0.0
 Requires-Dist: dkist-fits-specifications<5.0,>=4.0.0
-Requires-Dist: dkist-header-validator<6.0,>=5.0.0
+Requires-Dist: dkist-header-validator<6.0,>=5.3.0
 Requires-Dist: dkist-processing-core==7.0.1
 Requires-Dist: dkist-processing-pac<4.0,>=3.1
 Requires-Dist: dkist-service-configuration<5.0,>=4.2.0

dkist_processing_common-12.2.0rc2/changelog/272.feature.rst

@@ -0,0 +1 @@
+Add the framework for using Dataset Extras, that is, other data that can be included with the L1 FITS files generated by the regular pipeline. A new abstract class, WriteL1DatasetExtras, provides helper functionality for use in the instrument pipelines.

{dkist_processing_common-12.1.0rc1 → dkist_processing_common-12.2.0rc2}/dkist_processing_common/models/constants.py

@@ -285,18 +285,24 @@ class ConstantsBase:
     def dark_observing_program_execution_ids(self) -> list[str]:
         """Return the observing program execution ids constant for the dark task."""
         observing_programs = self._db_dict[BudName.dark_observing_program_execution_ids]
+        if isinstance(observing_programs, str):
+            observing_programs = [observing_programs]
         return list(observing_programs)
 
     @property
     def solar_gain_observing_program_execution_ids(self) -> list[str]:
         """Return the observing program execution ids constant for the solar_gain task."""
         observing_programs = self._db_dict[BudName.solar_gain_observing_program_execution_ids]
+        if isinstance(observing_programs, str):
+            observing_programs = [observing_programs]
         return list(observing_programs)
 
     @property
     def polcal_observing_program_execution_ids(self) -> list[str]:
         """Return the observing program execution ids constant."""
         observing_programs = self._db_dict[BudName.polcal_observing_program_execution_ids]
+        if isinstance(observing_programs, str):
+            observing_programs = [observing_programs]
         return list(observing_programs)
 
     @property

dkist_processing_common-12.2.0rc2/dkist_processing_common/models/extras.py

@@ -0,0 +1,35 @@
+"""Autocomplete access to dataset extra header sections."""
+
+from enum import StrEnum
+
+
+class DatasetExtraHeaderSection(StrEnum):
+    """Enum defining the possible header sections for dataset extras."""
+
+    common = "common"
+    aggregate = "aggregate"
+    iptask = "iptask"
+    gos = "gos"
+    wavecal = "wavecal"
+    atlas = "atlas"
+    test = "test"
+
+
+class DatasetExtraType(StrEnum):
+    """Enum defining options for dataset extra names."""
+
+    dark = "DARK"
+    background_light = "BACKGROUND LIGHT"
+    solar_gain = "SOLAR GAIN"
+    characteristic_spectra = "CHARACTERISTIC SPECTRA"
+    modulation_state_offsets = "MODULATION STATE OFFSETS"
+    beam_angles = "BEAM ANGLES"
+    spectral_curvature_shifts = "SPECTRAL CURVATURE SHIFTS"
+    wavelength_calibration_input_spectrum = "WAVELENGTH CALIBRATION INPUT SPECTRUM"
+    wavelength_calibration_reference_spectrum = "WAVELENGTH CALIBRATION REFERENCE SPECTRUM"
+    reference_wavelength_vector = "REFERENCE WAVELENGTH VECTOR"
+    demodulation_matrices = "DEMODULATION MATRICES"
+    polcal_as_science = "POLCAL AS SCIENCE"
+    bad_pixel_map = "BAD PIXEL MAP"
+    beam_offsets = "BEAM OFFSETS"
+    spectral_curvature_scales = "SPECTRAL CURVATURE SCALES"

{dkist_processing_common-12.1.0rc1 → dkist_processing_common-12.2.0rc2}/dkist_processing_common/models/tags.py

@@ -38,6 +38,8 @@ class StemName(StrEnum):
     dataset_inventory = "DATASET_INVENTORY"
     asdf = "ASDF"
     quality_report = "QUALITY_REPORT"
+    # Dataset extras
+    extra = "EXTRA"
 
 
 class Tag:
@@ -450,3 +452,14 @@ class Tag:
         An asdf tag
         """
         return cls.format_tag(StemName.asdf)
+
+    @classmethod
+    def extra(cls) -> str:
+        """
+        Return a dataset extra tag.
+
+        Returns
+        -------
+        A dataset extra tag
+        """
+        return cls.format_tag(StemName.extra)

{dkist_processing_common-12.1.0rc1 → dkist_processing_common-12.2.0rc2}/dkist_processing_common/parsers/id_bud.py

@@ -1,16 +1,17 @@
 """Base classes for ID bud parsing."""
 
+from collections import Counter
 from enum import StrEnum
 from typing import Callable
 from typing import Type
 
-from dkist_processing_common.models.flower_pot import SetStem
+from dkist_processing_common.models.flower_pot import ListStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 
 
-class ContributingIdsBud(SetStem):
+class ContributingIdsBud(ListStem):
     """Base class for contributing ID buds."""
 
     def __init__(self, constant_name: str, metadata_key: str | StrEnum):
@@ -35,13 +36,15 @@ class ContributingIdsBud(SetStem):
 
     def getter(self) -> tuple[str, ...]:
         """
-        Get all ids seen for any type of frame.
+        Get all ids seen for any type of frame, sorted by the number of appearances of that ID.
 
         Returns
         -------
         IDs from all types of frames
         """
-        return tuple(self.value_set)
+        counts = Counter(self.value_list)  # Count the number of appearances of each ID
+        sorted_ids = tuple(str(item) for item, count in counts.most_common())
+        return sorted_ids
 
 
 class TaskContributingIdsBud(ContributingIdsBud):

{dkist_processing_common-12.1.0rc1 → dkist_processing_common-12.2.0rc2}/dkist_processing_common/tasks/l1_output_data.py

@@ -2,18 +2,14 @@
 
 import logging
 from abc import ABC
-from itertools import chain
 from pathlib import Path
 from typing import Iterable
 
-from dkist_processing_common.codecs.quality import quality_data_decoder
 from dkist_processing_common.codecs.quality import quality_data_encoder
 from dkist_processing_common.models.message import CatalogFrameMessage
 from dkist_processing_common.models.message import CatalogFrameMessageBody
 from dkist_processing_common.models.message import CatalogObjectMessage
 from dkist_processing_common.models.message import CatalogObjectMessageBody
-from dkist_processing_common.models.message import CreateQualityReportMessage
-from dkist_processing_common.models.message import CreateQualityReportMessageBody
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.mixin.globus import GlobusMixin
 from dkist_processing_common.tasks.mixin.interservice_bus import InterserviceBusMixin
@@ -62,15 +58,19 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
         with self.telemetry_span("Upload quality data"):
             self.transfer_quality_data()
 
-        with self.telemetry_span("Upload science frames"):
+        with self.telemetry_span("Upload output frames"):
             self.transfer_output_frames()
 
     def transfer_output_frames(self):
-        """Create a Globus transfer for all output data."""
-        transfer_items = self.build_output_frame_transfer_list()
+        """Create a Globus transfer for all output data, as well as any available dataset extras."""
+        output_transfer_items = self.build_output_frame_transfer_list()
+        dataset_extra_transfer_items = self.build_dataset_extra_transfer_list()
+        transfer_items = output_transfer_items + dataset_extra_transfer_items
 
         logger.info(
             f"Preparing globus transfer {len(transfer_items)} items: "
+            f"{len(output_transfer_items)} output frames. "
+            f"{len(dataset_extra_transfer_items)} dataset extras. "
             f"recipe_run_id={self.recipe_run_id}. "
             f"transfer_items={transfer_items[:3]}..."
         )
@@ -246,19 +246,24 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
     def run(self) -> None:
         """Run method for this task."""
         with self.telemetry_span("Gather output data"):
-            frames = self.read(tags=self.output_frame_tags)
-            movies = self.read(tags=[Tag.output(), Tag.movie()])
+            frames = self.read(
+                tags=self.output_frame_tags
+            )  # frames is kept as a generator as it is much longer than the other file categories
+            extras = list(self.read(tags=self.extra_frame_tags))
+            movies = list(self.read(tags=[Tag.output(), Tag.movie()]))
             quality_data = self.read(tags=[Tag.output(), Tag.quality_data()])
         with self.telemetry_span("Create message objects"):
             messages = []
             messages += self.frame_messages(paths=frames)
             frame_message_count = len(messages)
+            messages += self.frame_messages(paths=extras)
+            extra_message_count = len(extras)
             messages += self.object_messages(paths=movies, object_type="MOVIE")
-            object_message_count = len(messages) - frame_message_count
+            object_message_count = len(movies)
             dataset_has_quality_data = self.dataset_has_quality_data
             if dataset_has_quality_data:
                 messages += self.object_messages(paths=quality_data, object_type="QDATA")
         with self.telemetry_span(
-            f"Publish messages: {frame_message_count = }, {object_message_count = }, {dataset_has_quality_data = }"
+            f"Publish messages: {frame_message_count = }, {extra_message_count = }, {object_message_count = }, {dataset_has_quality_data = }"
         ):
             self.interservice_bus_publish(messages=messages)

{dkist_processing_common-12.1.0rc1 → dkist_processing_common-12.2.0rc2}/dkist_processing_common/tasks/output_data_base.py

@@ -22,19 +22,23 @@ class OutputDataBase(WorkflowTaskBase, ABC):
         """Get the destination bucket."""
         return self.metadata_store_recipe_run.configuration.destination_bucket
 
-    def format_object_key(self, path: Path) -> str:
+    def format_object_key(self, path: Path, folder_modifier: str | None = None) -> str:
         """
         Convert output paths into object store keys.
 
         Parameters
         ----------
         path: the Path to convert
+        folder_modifier: optional folder name to insert into the path
 
         Returns
         -------
         formatted path in the object store
         """
-        object_key = self.destination_folder / Path(path.name)
+        if folder_modifier:
+            object_key = self.destination_folder / Path(folder_modifier) / Path(path.name)
+        else:
+            object_key = self.destination_folder / Path(path.name)
         return str(object_key)
 
     @property
@@ -52,6 +56,11 @@ class OutputDataBase(WorkflowTaskBase, ABC):
         """Tags that uniquely identify L1 fits frames i.e. the dataset-inventory-able frames."""
         return [Tag.output(), Tag.frame()]
 
+    @property
+    def extra_frame_tags(self) -> list[str]:
+        """Tags that uniquely identify dataset extra fits frames."""
+        return [Tag.output(), Tag.extra()]
+
 
 class TransferDataBase(OutputDataBase, ObjectStoreMixin, ABC):
     """Base class for transferring data from scratch to somewhere else."""
@@ -73,9 +82,21 @@ class TransferDataBase(OutputDataBase, ObjectStoreMixin, ABC):
         """Build a list of GlobusTransfer items corresponding to all OUTPUT (i.e., L1) frames."""
         science_frame_paths: list[Path] = list(self.read(tags=self.output_frame_tags))
 
+        return self.build_transfer_list(science_frame_paths)
+
+    def build_dataset_extra_transfer_list(self) -> list[GlobusTransferItem]:
+        """Build a list of GlobusTransfer items corresponding to all extra dataset files."""
+        extra_paths: list[Path] = list(self.read(tags=self.extra_frame_tags))
+
+        return self.build_transfer_list(paths=extra_paths, destination_folder_modifier="extra")
+
+    def build_transfer_list(
+        self, paths: list[Path], destination_folder_modifier: str | None = None
+    ) -> list[GlobusTransferItem]:
+        """Given a list of paths, build a list of GlobusTransfer items."""
         transfer_items = []
-        for p in science_frame_paths:
-            object_key = self.format_object_key(p)
+        for p in paths:
+            object_key = self.format_object_key(path=p, folder_modifier=destination_folder_modifier)
             destination_path = Path(self.destination_bucket, object_key)
             item = GlobusTransferItem(
                 source_path=p,

dkist_processing_common-12.2.0rc2/dkist_processing_common/tasks/write_extra.py

@@ -0,0 +1,333 @@
+"""Classes to support the generation of dataset extras."""
+
+import uuid
+from abc import ABC
+from abc import abstractmethod
+from datetime import datetime
+
+import numpy as np
+from astropy.io import fits
+from astropy.time import Time
+from dkist_fits_specifications.utils.formatter import reformat_dataset_extra_header
+from dkist_header_validator.spec_validators import spec_extras_validator
+
+from dkist_processing_common.codecs.fits import fits_hdulist_encoder
+from dkist_processing_common.models.extras import DatasetExtraHeaderSection
+from dkist_processing_common.models.extras import DatasetExtraType
+from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.models.task_name import TaskName
+from dkist_processing_common.tasks.write_l1_base import WriteL1Base
+
+
+class WriteL1DatasetExtras(WriteL1Base, ABC):
+    """Class supporting the construction of dataset extras."""
+
+    def dataset_extra_headers(
+        self,
+        filename: str,
+        task_type: TaskName,
+        extra_name: DatasetExtraType,
+        total_exposure: float | None = None,
+        readout_exposure: float | None = None,
+    ) -> dict:
+        """Provide common FITS header keys for dataset extras."""
+        # Build task specific header values
+        match task_type:
+            case TaskName.dark:
+                task_specific_observing_program_execution_id = (
+                    self.constants.dark_observing_program_execution_ids
+                )
+                task_specific_date_begin = self.constants.dark_date_begin
+                task_specific_date_end = self.constants.dark_date_end
+                task_specific_raw_frames_per_fpa = (
+                    0  # can be updated in construction of dataset extra if required
+                )
+                task_specific_telescope_tracking_mode = (
+                    "None"  # can be updated in construction of dataset extra if required
+                )
+                task_specific_coude_table_tracking_mode = (
+                    "None"  # can be updated in construction of dataset extra if required
+                )
+                task_specific_telescope_scanning_mode = (
+                    "None"  # can be updated in construction of dataset extra if required
+                )
+                task_specific_average_light_level = self.constants.dark_average_light_level
+                task_specific_average_telescope_elevation = (
+                    self.constants.dark_average_telescope_elevation
+                )
+                task_specific_average_coude_table_angle = (
+                    self.constants.dark_average_coude_table_angle
+                )
+                task_specific_average_telescope_azimuth = (
+                    self.constants.dark_average_telescope_azimuth
+                )
+                task_specific_gos_level3_status = self.constants.dark_gos_level3_status
+                task_specific_gos_level3_lamp_status = self.constants.dark_gos_level3_lamp_status
+                task_specific_gos_polarizer_status = self.constants.dark_gos_polarizer_status
+                task_specific_gos_polarizer_angle = self.constants.dark_gos_polarizer_angle
+                task_specific_gos_retarder_status = self.constants.dark_gos_retarder_status
+                task_specific_gos_retarder_angle = self.constants.dark_gos_retarder_angle
+                task_specific_gos_level0_status = self.constants.dark_gos_level0_status
+            case TaskName.solar_gain:
+                task_specific_observing_program_execution_id = (
+                    self.constants.solar_gain_observing_program_execution_ids
+                )
+
+                task_specific_date_begin = self.constants.solar_gain_date_begin
+                task_specific_date_end = self.constants.solar_gain_date_end
+                task_specific_raw_frames_per_fpa = self.constants.solar_gain_num_raw_frames_per_fpa
+                task_specific_telescope_tracking_mode = (
+                    self.constants.solar_gain_telescope_tracking_mode
+                )
+                task_specific_coude_table_tracking_mode = (
+                    self.constants.solar_gain_coude_table_tracking_mode
+                )
+                task_specific_telescope_scanning_mode = (
+                    self.constants.solar_gain_telescope_scanning_mode
+                )
+                task_specific_average_light_level = self.constants.solar_gain_average_light_level
+                task_specific_average_telescope_elevation = (
+                    self.constants.solar_gain_average_telescope_elevation
+                )
+                task_specific_average_coude_table_angle = (
+                    self.constants.solar_gain_average_coude_table_angle
+                )
+                task_specific_average_telescope_azimuth = (
+                    self.constants.solar_gain_average_telescope_azimuth
+                )
+                task_specific_gos_level3_status = self.constants.solar_gain_gos_level3_status
+                task_specific_gos_level3_lamp_status = (
+                    self.constants.solar_gain_gos_level3_lamp_status
+                )
+                task_specific_gos_polarizer_status = self.constants.solar_gain_gos_polarizer_status
+                task_specific_gos_polarizer_angle = self.constants.solar_gain_gos_polarizer_angle
+                task_specific_gos_retarder_status = self.constants.solar_gain_gos_retarder_status
+                task_specific_gos_retarder_angle = self.constants.solar_gain_gos_retarder_angle
+                task_specific_gos_level0_status = self.constants.solar_gain_gos_level0_status
+            case TaskName.polcal:
+                task_specific_observing_program_execution_id = (
+                    self.constants.polcal_observing_program_execution_ids
+                )
+
+                task_specific_date_begin = self.constants.polcal_date_begin
+                task_specific_date_end = self.constants.polcal_date_end
+                task_specific_raw_frames_per_fpa = self.constants.polcal_num_raw_frames_per_fpa
+                task_specific_telescope_tracking_mode = (
+                    self.constants.polcal_telescope_tracking_mode
+                )
+                task_specific_coude_table_tracking_mode = (
+                    self.constants.polcal_coude_table_tracking_mode
+                )
+                task_specific_telescope_scanning_mode = (
+                    self.constants.polcal_telescope_scanning_mode
+                )
+                task_specific_average_light_level = self.constants.polcal_average_light_level
+                task_specific_average_telescope_elevation = (
+                    self.constants.polcal_average_telescope_elevation
+                )
+                task_specific_average_coude_table_angle = (
+                    self.constants.polcal_average_coude_table_angle
+                )
+                task_specific_average_telescope_azimuth = (
+                    self.constants.polcal_average_telescope_azimuth
+                )
+                task_specific_gos_level3_status = None
+                task_specific_gos_level3_lamp_status = None
+                task_specific_gos_polarizer_status = None
+                task_specific_gos_polarizer_angle = None
+                task_specific_gos_retarder_status = None
+                task_specific_gos_retarder_angle = None
+                task_specific_gos_level0_status = None
+            case _:
+                raise ValueError(f"Unsupported task type {task_type}")
+
+        start_datetime = datetime.fromisoformat(task_specific_date_begin)
+        end_datetime = datetime.fromisoformat(task_specific_date_end)
+
+        dataset_extra_header = {
+            DatasetExtraHeaderSection.common: {
+                "BUNIT": "count",
+                "DATE": Time.now().fits,
+                "DATE-BEG": task_specific_date_begin,
+                "DATE-END": task_specific_date_end,
+                "TELAPSE": (end_datetime - start_datetime).total_seconds(),
+                "DATE-AVG": (start_datetime + (end_datetime - start_datetime) / 2).isoformat(),
+                "TIMESYS": "UTC",
+                "ORIGIN": "National Solar Observatory",
+                "TELESCOP": "Daniel K. Inouye Solar Telescope",
+                "OBSRVTRY": "Haleakala High Altitude Observatory Site",
+                "NETWORK": "NSF-DKIST",
+                "INSTRUME": self.constants.instrument,
+                "OBJECT": "unknown",
+                "CAM_ID": self.constants.camera_id,
+                "CAMERA": self.constants.camera_name,
+                "BITDEPTH": self.constants.camera_bit_depth,
+                "XPOSURE": total_exposure,
+                "TEXPOSUR": readout_exposure,
+                "HWBIN1": self.constants.hardware_binning_x,
+                "HWBIN2": self.constants.hardware_binning_y,
+                "SWBIN1": self.constants.software_binning_x,
+                "SWBIN2": self.constants.software_binning_y,
+                "NSUMEXP": task_specific_raw_frames_per_fpa,
+                "DSETID": self.constants.dataset_id,
+                "PROCTYPE": "L1_EXTRA",
+                "RRUNID": self.recipe_run_id,
+                "RECIPEID": self.metadata_store_recipe_run.recipeInstance.recipeId,
+                "RINSTID": self.metadata_store_recipe_run.recipeInstanceId,
+                "FILENAME": filename,
+                "HEAD_URL": "",
+                "INFO_URL": self.docs_base_url,
+                "CAL_URL": "",
+                "CALVERS": self.version_from_module_name(),
+                "IDSPARID": (
+                    parameters.inputDatasetPartId
+                    if (parameters := self.metadata_store_input_dataset_parameters)
+                    else None
+                ),
+                "IDSOBSID": (
+                    observe_frames.inputDatasetPartId
+                    if (observe_frames := self.metadata_store_input_dataset_observe_frames)
+                    else None
+                ),
+                "IDSCALID": (
+                    calibration_frames.inputDatasetPartId
+                    if (calibration_frames := self.metadata_store_input_dataset_calibration_frames)
+                    else None
+                ),
+                "WKFLVERS": self.workflow_version,
+                "WKFLNAME": self.workflow_name,
+                "MANPROCD": self.workflow_had_manual_intervention,
+                "FILE_ID": uuid.uuid4().hex,
+                "OBSPR_ID": task_specific_observing_program_execution_id[
+                    0
+                ],  # The OP IDs are stored sorted by number of appearances of each OP ID in the source task type frames
+                "EXTOBSID": ",".join(task_specific_observing_program_execution_id[1:]),
+                "EXPER_ID": self.constants.experiment_id,
+                "PROP_ID": self.constants.proposal_id,
+                "HLSVERS": self.constants.hls_version,
+                "LINEWAV": self.constants.wavelength,
+                "TELTRACK": (
+                    task_specific_telescope_tracking_mode if task_type != TaskName.dark else None
+                ),
+                "TTBLTRCK": (
+                    task_specific_coude_table_tracking_mode if task_type != TaskName.dark else None
+                ),
+                "TELSCAN": (
+                    task_specific_telescope_scanning_mode if task_type != TaskName.dark else None
+                ),
+                "EXTNAME": extra_name,
+            },
+            DatasetExtraHeaderSection.aggregate: {
+                "AVGLLVL": task_specific_average_light_level,
+                "ATELEVAT": task_specific_average_telescope_elevation,
+                "ATTBLANG": task_specific_average_coude_table_angle,
+                "ATAZIMUT": task_specific_average_telescope_azimuth,
+            },
+            DatasetExtraHeaderSection.iptask: {
+                "IPTASK": "GAIN" if "GAIN" in task_type else task_type,
+            },
+            DatasetExtraHeaderSection.gos: {
+                "LVL3STAT": task_specific_gos_level3_status,
+                "LAMPSTAT": task_specific_gos_level3_lamp_status,
+                "LVL2STAT": task_specific_gos_polarizer_status,
+                "POLANGLE": task_specific_gos_polarizer_angle,
+                "LVL1STAT": task_specific_gos_retarder_status,
+                "RETANGLE": task_specific_gos_retarder_angle,
+                "LVL0STAT": task_specific_gos_level0_status,
+            },
+        }
+
+        # Remove specific headers from dark frames as they don't constants to fill them
+        if task_type == TaskName.dark:
+            for key in ["TELTRACK", "TTBLTRCK", "TELSCAN"]:
+                del dataset_extra_header[DatasetExtraHeaderSection.common][key]
+
+        # Remove specific headers from polcal frames as they don't have constants to fill them
+        if task_type == TaskName.polcal:
+            for key in [
+                "LVL3STAT",
+                "LAMPSTAT",
+                "LVL2STAT",
+                "POLANGLE",
+                "LVL1STAT",
+                "RETANGLE",
+                "LVL0STAT",
+            ]:
+                del dataset_extra_header[DatasetExtraHeaderSection.gos][key]
+
+        return dataset_extra_header
+
+    def build_dataset_extra_header(
+        self,
+        sections: list[DatasetExtraHeaderSection],
+        filename: str,
+        task_type: TaskName,
+        extra_name: DatasetExtraType,
+        total_exposure: float | None = None,
+        readout_exposure: float | None = None,
+    ) -> fits.Header:
+        """Build FITS header for dataset extra file."""
+        header = fits.Header()
+        all_section_headers = self.dataset_extra_headers(
+            filename=filename,
+            task_type=task_type,
+            total_exposure=total_exposure,
+            readout_exposure=readout_exposure,
+            extra_name=extra_name,
+        )
+        for section in sections:
+            header.update(all_section_headers[section].items())
+        return header
+
+    def format_extra_filename(self, extra_name: DatasetExtraType | str, detail: str | None = None):
+        """Format the filename of dataset extras for consistency."""
+        base_filename = f"{self.constants.instrument}_{self.constants.dataset_id}_{extra_name.replace(' ', '-')}"
+        if detail:
+            base_filename += "_" + detail
+        filename_counter = str(self.filename_counter.increment(base_filename))
+        return f"{base_filename}_{filename_counter}.fits"
+
+    def assemble_and_write_dataset_extra(
+        self,
+        data: np.ndarray | list[np.ndarray],
+        header: fits.Header | list[fits.Header],
+        filename: str,
+    ):
+        """Given the data and header information, write the dataset extra."""
+        if isinstance(data, list) and isinstance(header, list):
+            if len(data) != len(header):
+                raise ValueError(
+                    f"{len(data)} data arrays were provided with {len(header)} headers. These must be equal."
+                )
+        if isinstance(data, np.ndarray):
+            data = [data]
+        if isinstance(header, fits.Header):
+            header = [header]
+        hdus = [fits.PrimaryHDU()]  # The first HDU in the list is an empty PrimaryHDU
+        for i, data_array in enumerate(data):
+            tile_size = self.compute_tile_size_for_array(data_array)
+            hdu = fits.CompImageHDU(header=header[i], data=data_array, tile_shape=tile_size)
+            formatted_header = reformat_dataset_extra_header(hdu.header)
+            hdu = fits.CompImageHDU(header=formatted_header, data=hdu.data, tile_shape=tile_size)
+            hdus.append(hdu)
+        self.write(
+            data=fits.HDUList(hdus),
+            tags=[Tag.extra(), Tag.output()],
+            encoder=fits_hdulist_encoder,
+            relative_path=filename,
+        )
+        self.update_framevol(relative_path=filename)
+
+        # Check that the written file passes spec 214 validation if requested
+        if self.validate_l1_on_write:
+            spec_extras_validator.validate(self.scratch.absolute_path(filename), extra=False)
+
+    @abstractmethod
+    def run(self) -> None:
+        """
+        For each dataset extra.
+
+        * Gather the source data in whatever manner is necessary
+        * Build a header using the `build_dataset_extra_header` method to help with header construction
+        * Write the dataset extra using `assemble_and_write_dataset_extra()`
+        """