dkist-processing-common 10.5.4__py3-none-any.whl → 12.1.0rc1__py3-none-any.whl

dkist_processing_common/tasks/mixin/quality/_metrics.py

@@ -2,6 +2,7 @@
 
 These classes should not be directly mixed in to anything. They are pre-mixed into the top-level QualityMixin
 """
+
 import copy
 import json
 import logging
@@ -12,14 +13,19 @@ from typing import Any
 from typing import Iterable
 from typing import Literal
 
+import astropy.units as u
 import numpy as np
+from astropy.wcs import WCS
 from dkist_processing_pac.fitter.fitter_parameters import CU_PARAMS
 from dkist_processing_pac.fitter.fitter_parameters import GLOBAL_PARAMS
 from dkist_processing_pac.fitter.fitter_parameters import TELESCOPE_PARAMS
 from dkist_processing_pac.fitter.fitting_core import compare_I
 from dkist_processing_pac.fitter.polcal_fitter import PolcalFitter
 from pandas import DataFrame
+from solar_wavelength_calibration.fitter.wavelength_fitter import FitResult
 
+from dkist_processing_common.codecs.json import json_decoder
+from dkist_processing_common.models.fried_parameter import r0_valid
 from dkist_processing_common.models.metric_code import MetricCode
 from dkist_processing_common.models.quality import EfficiencyHistograms
 from dkist_processing_common.models.quality import ModulationMatrixHistograms
@@ -28,6 +34,7 @@ from dkist_processing_common.models.quality import PlotHistogram
 from dkist_processing_common.models.quality import PlotRaincloud
 from dkist_processing_common.models.quality import ReportMetric
 from dkist_processing_common.models.quality import SimpleTable
+from dkist_processing_common.models.quality import VerticalMultiPanePlot2D
 from dkist_processing_common.models.tags import Tag
 
 logger = logging.getLogger(__name__)
@@ -203,7 +210,7 @@ class _SimplePlotQualityMixin:
         return warnings
 
     def quality_store_ao_status_and_fried_parameter(
-        self, datetimes: list[str], values: list[list[bool, float]]
+        self, datetimes: list[str], values: list[list[bool | float]]
     ):
         """
         Collect and store datetime / value pairs for the boolean AO status and Fried parameter.
@@ -213,23 +220,27 @@ class _SimplePlotQualityMixin:
         Because of how L1Metric.has_metric works, empty lists will not be passed to this method.
         However, because of how L1Metric.store_metric works, one or both values can be None.
         """
-        ao_values = [value[0] for value in values]
-        ao_not_none = [ao for ao in ao_values if ao is not None]
+        ao_lock_values = [value[0] for value in values]
+        ao_not_none = [ao for ao in ao_lock_values if ao is not None]
         if len(ao_not_none) != 0:
             self._record_values(values=ao_not_none, tags=Tag.quality(MetricCode.ao_status))
         fried_values = [value[1] for value in values]
-        fried_values_not_none_ao_locked = [
-            r0 for ao, r0 in zip(ao_values, fried_values) if (ao is True) and (r0 is not None)
-        ]
-        datetimes_fried_not_none_ao_locked = [
-            dt
-            for dt, ao, r0 in zip(datetimes, ao_values, fried_values)
-            if (ao is True) and (r0 is not None)
-        ]
-        if len(fried_values_not_none_ao_locked) != 0:
+        ao_oob_values = [value[2] for value in values]
+        fried_values_to_plot = []
+        datetimes_to_plot = []
+        # For each set of input data, check if the r0 is considered valid based on all data
+        for i in range(len(fried_values)):
+            if r0_valid(
+                r0=fried_values[i],
+                ao_lock=ao_lock_values[i],
+                num_out_of_bounds_ao_values=ao_oob_values[i],
+            ):
+                fried_values_to_plot.append(fried_values[i])
+                datetimes_to_plot.append(datetimes[i])
+        if len(fried_values_to_plot) != 0:
             self._record_2d_plot_values(
-                x_values=datetimes_fried_not_none_ao_locked,
-                y_values=fried_values_not_none_ao_locked,
+                x_values=datetimes_to_plot,
+                y_values=fried_values_to_plot,
                 tags=Tag.quality(MetricCode.fried_parameter),
             )
 
@@ -1319,3 +1330,144 @@ class _PolcalQualityMixin:
         base_str += " bins."
 
         return base_str
+
+
+class _WavecalQualityMixin:
+    """Mixin class supporting the recording and building of wavecal-related metrics."""
+
+    def quality_store_wavecal_results(
+        self,
+        *,
+        input_wavelength: u.Quantity,
+        input_spectrum: np.ndarray,
+        fit_result: FitResult,
+        weights: None | np.ndarray = None,
+    ):
+        """
+        Store the results of a wavelength solution fit.
+
+        Namely, save the:
+
+        * Input spectrum and wavelength
+        * Best-fit combined atlas spectrum
+        * Best-fit wavelength vector
+        * Fit residuals
+
+        Note that the residuals are the *unweighed* residuals.
+        """
+        weight_data = np.ones(input_wavelength.size) if weights is None else weights
+        prepared_weights = fit_result.prepared_weights
+        residuals = fit_result.minimizer_result.residual / prepared_weights
+        residuals[~np.isfinite(residuals)] = 0.0
+        normalized_residuals = residuals / input_spectrum
+
+        best_fit_atlas = fit_result.best_fit_atlas
+        best_fit_wavelength = fit_result.best_fit_wavelength_vector
+
+        finite_idx = (
+            np.isfinite(input_wavelength)
+            * np.isfinite(input_spectrum)
+            * np.isfinite(best_fit_wavelength)
+            * np.isfinite(best_fit_atlas)
+            * np.isfinite(normalized_residuals)
+            * np.isfinite(weight_data)
+        )
+
+        data = {
+            "input_wavelength_nm": input_wavelength.to_value(u.nm)[finite_idx].tolist(),
+            "input_spectrum": input_spectrum[finite_idx].tolist(),
+            "best_fit_wavelength_nm": best_fit_wavelength[finite_idx].tolist(),
+            "best_fit_atlas": best_fit_atlas[finite_idx].tolist(),
+            "normalized_residuals": normalized_residuals[finite_idx].tolist(),
+            "weights": None if weights is None else weight_data[finite_idx].tolist(),
+        }
+
+        self._record_values(values=data, tags=[Tag.quality(MetricCode.wavecal_fit)])
+
+    def quality_build_wavecal_results(self) -> dict:
+        """Build a ReportMetric containing a multi-pane plot showing the fit spectra and residuals."""
+        data = next(self.read(tags=[Tag.quality(MetricCode.wavecal_fit)], decoder=json_decoder))
+
+        input_wave_list = data["input_wavelength_nm"]
+        input_spectrum_list = data["input_spectrum"]
+        best_fit_wave_list = data["best_fit_wavelength_nm"]
+        best_fit_atlas_list = data["best_fit_atlas"]
+        residuals_list = data["normalized_residuals"]
+        weights = data["weights"]
+
+        fit_series = {
+            "Best Fit Observations": [best_fit_wave_list, input_spectrum_list],
+            "Input Spectrum": [input_wave_list, input_spectrum_list],
+            "Best Fit Atlas": [best_fit_wave_list, best_fit_atlas_list],
+        }
+
+        # Set the colors and zorder here manually because the JSON-ization of the quality data means we can't be sure of
+        # the order these will be plotted in and thus can't rely on the default color-cycler in `dkist-quality`.
+        fit_plot_kwargs = {
+            "Best Fit Observations": {
+                "ls": "-",
+                "lw": 4,
+                "alpha": 0.8,
+                "ms": 0,
+                "color": "#FAA61C",
+                "zorder": 2,
+            },
+            "Input Spectrum": {"ls": "-", "alpha": 0.4, "ms": 0, "color": "#1E317A", "zorder": 2.1},
+            "Best Fit Atlas": {"color": "k", "ls": "-", "ms": 0, "zorder": 2.2},
+        }
+
+        fit_plot = Plot2D(
+            xlabel="Wavelength [nm]",
+            ylabel="Signal",
+            series_data=fit_series,
+            plot_kwargs=fit_plot_kwargs,
+            sort_series=False,
+        )
+
+        residuals_series = {"Residuals": [best_fit_wave_list, residuals_list]}
+        residuals_plot_kwargs = {"Residuals": {"ls": "-", "color": "k", "ms": 0}}
+
+        y_min = np.nanpercentile(residuals_list, 2)
+        y_max = np.nanpercentile(residuals_list, 98)
+        y_range = y_max - y_min
+        y_min -= 0.1 * y_range
+        y_max += 0.1 * y_range
+        residuals_plot = Plot2D(
+            xlabel="Wavelength [nm]",
+            ylabel=r"$\frac{\mathrm{Obs - Atlas}}{\mathrm{Obs}}$",
+            series_data=residuals_series,
+            plot_kwargs=residuals_plot_kwargs,
+            ylim=(y_min, y_max),
+        )
+
+        plot_list = [fit_plot, residuals_plot]
+        height_ratios = [1.5, 1.0]
+        if weights is not None:
+            weight_series = {"Weights": [best_fit_wave_list, weights]}
+            weight_plot_kwargs = {"Weights": {"ls": "-", "color": "k", "ms": 0}}
+            weight_plot = Plot2D(
+                xlabel="Wavelength [nm]",
+                ylabel="Fit Weights",
+                series_data=weight_series,
+                plot_kwargs=weight_plot_kwargs,
+            )
+            plot_list.append(weight_plot)
+            height_ratios.append(1.0)
+
+        full_plot = VerticalMultiPanePlot2D(
+            top_to_bottom_plot_list=plot_list,
+            match_x_axes=True,
+            no_gap=True,
+            top_to_bottom_height_ratios=height_ratios,
+        )
+
+        metric = ReportMetric(
+            name="Wavelength Calibration Results",
+            description="These plots show the wavelength solution computed based on fits to a Solar FTS atlas. "
+            "The top plot shows the input and best-fit spectra along with the best-fit atlas, which is "
+            "a combination of Solar and Telluric spectra. The bottom plot shows the fit residuals.",
+            metric_code=MetricCode.wavecal_fit,
+            multi_plot_data=full_plot,
+        )
+
+        return metric.model_dump()

dkist_processing_common/tasks/output_data_base.py

@@ -1,4 +1,5 @@
 """Base class that supports common output data methods and paths."""
+
 import logging
 from abc import ABC
 from abc import abstractmethod
@@ -19,7 +20,7 @@ class OutputDataBase(WorkflowTaskBase, ABC):
     @cached_property
     def destination_bucket(self) -> str:
         """Get the destination bucket."""
-        return self.metadata_store_recipe_run_configuration().get("destination_bucket", "data")
+        return self.metadata_store_recipe_run.configuration.destination_bucket
 
     def format_object_key(self, path: Path) -> str:
         """
@@ -57,10 +58,10 @@ class TransferDataBase(OutputDataBase, ObjectStoreMixin, ABC):
 
     def run(self) -> None:
         """Transfer the data and cleanup any folders."""
-        with self.apm_task_step("Transfer objects"):
+        with self.telemetry_span("Transfer objects"):
             self.transfer_objects()
 
-        with self.apm_task_step("Remove folder objects"):
+        with self.telemetry_span("Remove folder objects"):
             self.remove_folder_objects()
 
     @abstractmethod

dkist_processing_common/tasks/parse_l0_input_data.py

@@ -20,7 +20,7 @@ that makes the rest of the pipeline easy to write.
 In other words, we can find exactly the frame we need (tags) and, once we have it, we never need to look
 at a different frame to get information (constants).
 """
-import json
+
 import logging
 from abc import ABC
 from abc import abstractmethod
@@ -28,24 +28,33 @@ from typing import TypeVar
 
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.models.constants import BudName
+from dkist_processing_common.models.fits_access import MetadataKey
 from dkist_processing_common.models.flower_pot import FlowerPot
 from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.flower_pot import Thorn
 from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.models.task_name import TaskName
+from dkist_processing_common.parsers.average_bud import TaskAverageBud
 from dkist_processing_common.parsers.experiment_id_bud import ContributingExperimentIdsBud
 from dkist_processing_common.parsers.experiment_id_bud import ExperimentIdBud
+from dkist_processing_common.parsers.lookup_bud import TaskTimeLookupBud
+from dkist_processing_common.parsers.observing_program_id_bud import (
+    TaskContributingObservingProgramExecutionIdsBud,
+)
 from dkist_processing_common.parsers.proposal_id_bud import ContributingProposalIdsBud
 from dkist_processing_common.parsers.proposal_id_bud import ProposalIdBud
+from dkist_processing_common.parsers.task import parse_header_ip_task_with_gains
 from dkist_processing_common.parsers.time import AverageCadenceBud
 from dkist_processing_common.parsers.time import MaximumCadenceBud
 from dkist_processing_common.parsers.time import MinimumCadenceBud
+from dkist_processing_common.parsers.time import TaskDateBeginBud
 from dkist_processing_common.parsers.time import TaskExposureTimesBud
 from dkist_processing_common.parsers.time import TaskReadoutExpTimesBud
 from dkist_processing_common.parsers.time import VarianceCadenceBud
+from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
 from dkist_processing_common.parsers.unique_bud import UniqueBud
 from dkist_processing_common.tasks.base import WorkflowTaskBase
 
-
 __all__ = [
     "ParseL0InputDataBase",
     "ParseDataBase",
@@ -58,10 +67,254 @@ logger = logging.getLogger(__name__)
 S = TypeVar("S", bound=Stem)
 
 
+def dataset_extra_bud_factory() -> list[S]:
+    """Provide constant buds for use in dataset extras."""
+    return [
+        UniqueBud(constant_name=BudName.camera_id, metadata_key=MetadataKey.camera_id),
+        UniqueBud(constant_name=BudName.camera_name, metadata_key=MetadataKey.camera_name),
+        UniqueBud(
+            constant_name=BudName.camera_bit_depth, metadata_key=MetadataKey.camera_bit_depth
+        ),
+        UniqueBud(
+            constant_name=BudName.hardware_binning_x, metadata_key=MetadataKey.hardware_binning_x
+        ),
+        UniqueBud(
+            constant_name=BudName.hardware_binning_y, metadata_key=MetadataKey.hardware_binning_x
+        ),
+        UniqueBud(
+            constant_name=BudName.software_binning_x, metadata_key=MetadataKey.software_binning_x
+        ),
+        UniqueBud(
+            constant_name=BudName.software_binning_y, metadata_key=MetadataKey.software_binning_y
+        ),
+        UniqueBud(
+            constant_name=BudName.hls_version,
+            metadata_key=MetadataKey.hls_version,
+        ),
+        TaskContributingObservingProgramExecutionIdsBud(
+            constant_name=BudName.dark_observing_program_execution_ids,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskContributingObservingProgramExecutionIdsBud(
+            constant_name=BudName.solar_gain_observing_program_execution_ids,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskContributingObservingProgramExecutionIdsBud(
+            constant_name=BudName.polcal_observing_program_execution_ids,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskTimeLookupBud(
+            constant_name=BudName.dark_num_raw_frames_per_fpa,
+            key_metadata_key=MetadataKey.sensor_readout_exposure_time_ms,
+            value_metadata_key=MetadataKey.num_raw_frames_per_fpa,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_num_raw_frames_per_fpa,
+            metadata_key=MetadataKey.num_raw_frames_per_fpa,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_num_raw_frames_per_fpa,
+            metadata_key=MetadataKey.num_raw_frames_per_fpa,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_telescope_tracking_mode,
+            metadata_key=MetadataKey.telescope_tracking_mode,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_telescope_tracking_mode,
+            metadata_key=MetadataKey.telescope_tracking_mode,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_coude_table_tracking_mode,
+            metadata_key=MetadataKey.coude_table_tracking_mode,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_coude_table_tracking_mode,
+            metadata_key=MetadataKey.coude_table_tracking_mode,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_telescope_scanning_mode,
+            metadata_key=MetadataKey.telescope_scanning_mode,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_telescope_scanning_mode,
+            metadata_key=MetadataKey.telescope_scanning_mode,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_level3_status,
+            metadata_key=MetadataKey.gos_level3_status,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_level3_status,
+            metadata_key=MetadataKey.gos_level3_status,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_level3_lamp_status,
+            metadata_key=MetadataKey.gos_level3_lamp_status,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_level3_lamp_status,
+            metadata_key=MetadataKey.gos_level3_lamp_status,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_polarizer_status,
+            metadata_key=MetadataKey.gos_polarizer_status,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_polarizer_status,
+            metadata_key=MetadataKey.gos_polarizer_status,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_polarizer_angle,
+            metadata_key=MetadataKey.gos_polarizer_angle,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_polarizer_angle,
+            metadata_key=MetadataKey.gos_polarizer_angle,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_retarder_status,
+            metadata_key=MetadataKey.gos_retarder_status,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_retarder_status,
+            metadata_key=MetadataKey.gos_retarder_status,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_retarder_angle,
+            metadata_key=MetadataKey.gos_retarder_angle,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_retarder_angle,
+            metadata_key=MetadataKey.gos_retarder_angle,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_level0_status,
+            metadata_key=MetadataKey.gos_level0_status,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_level0_status,
+            metadata_key=MetadataKey.gos_level0_status,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.dark_average_light_level,
+            metadata_key=MetadataKey.light_level,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.solar_gain_average_light_level,
+            metadata_key=MetadataKey.light_level,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.polcal_average_light_level,
+            metadata_key=MetadataKey.light_level,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.dark_average_telescope_elevation,
+            metadata_key=MetadataKey.elevation,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.solar_gain_average_telescope_elevation,
+            metadata_key=MetadataKey.elevation,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.polcal_average_telescope_elevation,
+            metadata_key=MetadataKey.elevation,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.dark_average_coude_table_angle,
+            metadata_key=MetadataKey.table_angle,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.solar_gain_average_coude_table_angle,
+            metadata_key=MetadataKey.table_angle,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.polcal_average_coude_table_angle,
+            metadata_key=MetadataKey.table_angle,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.dark_average_telescope_azimuth,
+            metadata_key=MetadataKey.azimuth,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.solar_gain_average_telescope_azimuth,
+            metadata_key=MetadataKey.azimuth,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.polcal_average_telescope_azimuth,
+            metadata_key=MetadataKey.azimuth,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskDateBeginBud(
+            constant_name=BudName.dark_date_begin,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskDateBeginBud(
+            constant_name=BudName.solar_gain_date_begin,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskDateBeginBud(
+            constant_name=BudName.polcal_date_begin,
+            ip_task_types=TaskName.polcal,
+        ),
+    ]
+
+
 def default_constant_bud_factory() -> list[S]:
     """Provide default constant buds for use in common parsing tasks."""
-    return [
-        UniqueBud(constant_name=BudName.instrument.value, metadata_key="instrument"),
+    return dataset_extra_bud_factory() + [
+        UniqueBud(constant_name=BudName.instrument, metadata_key=MetadataKey.instrument),
         ProposalIdBud(),
         ContributingProposalIdsBud(),
         ExperimentIdBud(),
@@ -70,8 +323,10 @@ def default_constant_bud_factory() -> list[S]:
         MaximumCadenceBud(),
         MinimumCadenceBud(),
         VarianceCadenceBud(),
-        TaskExposureTimesBud(stem_name=BudName.dark_exposure_times.value, ip_task_type="dark"),
-        TaskReadoutExpTimesBud(stem_name=BudName.dark_readout_exp_times.value, ip_task_type="dark"),
+        TaskExposureTimesBud(stem_name=BudName.dark_exposure_times, ip_task_types=TaskName.dark),
+        TaskReadoutExpTimesBud(
+            stem_name=BudName.dark_readout_exp_times, ip_task_types=TaskName.dark
+        ),
     ]
 
 
@@ -100,21 +355,25 @@ class ParseDataBase(WorkflowTaskBase, ABC):
 
     @property
     @abstractmethod
-    def tags_for_input_frames(self) -> list[Tag]:
+    def tags_for_input_frames(self) -> list[str]:
         """Define the tags for the data that will be parsed."""
 
+    def pre_run(self) -> None:
+        """Execute pre-task setup."""
+        self.outer_loop_progress.total = self.scratch.count_all(tags=self.tags_for_input_frames)
+
     def run(self) -> None:
         """Run method for this task."""
-        with self.apm_task_step("Check that input frames exist"):
+        with self.telemetry_span("Check that input frames exist"):
             self.check_input_frames()
 
-        with self.apm_task_step("Ingest all input files"):
+        with self.telemetry_span("Ingest all input files"):
             tag_pot, constant_pot = self.make_flower_pots()
 
-        with self.apm_task_step("Update constants"):
+        with self.telemetry_span("Update constants"):
             self.update_constants(constant_pot)
 
-        with self.apm_task_step("Tag files"):
+        with self.telemetry_span("Tag files"):
             self.tag_petals(tag_pot)
 
     def make_flower_pots(self) -> tuple[FlowerPot, FlowerPot]:
@@ -125,6 +384,7 @@ class ParseDataBase(WorkflowTaskBase, ABC):
         constant_pot.stems += self.constant_buds
 
         for fits_obj in self.input_frames:
+            self.outer_loop_progress.increment()
             filepath = fits_obj.name
             tag_pot.add_dirt(filepath, fits_obj)
             constant_pot.add_dirt(filepath, fits_obj)
@@ -137,6 +397,8 @@ class ParseDataBase(WorkflowTaskBase, ABC):
         return self.read(
             tags=self.tags_for_input_frames,
             decoder=fits_access_decoder,
+            checksum=False,
+            disable_image_compression=True,
             fits_access_class=self.fits_parsing_class,
         )
 
@@ -158,9 +420,9 @@ class ParseDataBase(WorkflowTaskBase, ABC):
         None
         """
         for stem in constant_pot:
-            with self.apm_processing_step(f"Setting value of constant {stem.stem_name}"):
-                if len(stem.petals) == 0:
-                    # There are no petals so nothing to do
+            with self.telemetry_span(f"Setting value of constant {stem.stem_name}"):
+                if not stem.can_be_picked:
+                    # Nothing to do
                     continue
                 if stem.bud.value is Thorn:
                     # Must've been a picky bud that passed. We don't want to pick it because it has no value
@@ -183,7 +445,7 @@ class ParseDataBase(WorkflowTaskBase, ABC):
         None
         """
         for stem in tag_pot:
-            with self.apm_processing_step(f"Applying {stem.stem_name} tag to files"):
+            with self.telemetry_span(f"Applying {stem.stem_name} tag to files"):
                 for petal in stem.petals:
                     tag = Tag.format_tag(stem.stem_name, petal.value)
                     for path in petal.keys: