dkist-processing-trend 0.1.0__py3-none-any.whl

dkist_processing_trend/tasks/write_trend.py

@@ -0,0 +1,663 @@
+"""Task class for writing trend output files."""
+
+import uuid
+from typing import Type
+
+import numpy as np
+from astropy.io import fits
+from astropy.time import Time
+from dkist_fits_specifications import __version__ as spec_version
+from dkist_fits_specifications.utils.formatter import reformat_trend_header
+from dkist_processing_common.codecs.fits import fits_array_decoder
+from dkist_processing_common.codecs.fits import fits_hdulist_encoder
+from dkist_processing_common.tasks.write_l1_base import WriteL1Base
+from dkist_processing_pac.fitter.fitter_parameters import PolcalDresserParameters
+
+from dkist_processing_trend.models.constants import TrendConstants
+from dkist_processing_trend.models.fit_options import FitOptions
+from dkist_processing_trend.models.instrument import InstrumentName
+from dkist_processing_trend.models.instrument_options import CryonirspInstrumentOptions
+from dkist_processing_trend.models.instrument_options import DlnirspInstrumentOptions
+from dkist_processing_trend.models.instrument_options import InstrumentOptionsBase
+from dkist_processing_trend.models.instrument_options import VispInstrumentOptions
+from dkist_processing_trend.models.parameters import TrendParameters
+from dkist_processing_trend.models.tags import TrendTag
+
+__all__ = ["WriteTrendOutputs"]
+
+
+class WriteTrendOutputs(WriteL1Base):
+    """Task for writing trend output files.
+
+    Parameters
+    ----------
+    recipe_run_id
+        id of the recipe run used to identify the workflow run this task is part of
+
+    workflow_name
+        name of the workflow to which this instance of the task belongs
+
+    workflow_version
+        version of the workflow to which this instance of the task belongs
+    """
+
+    constants: TrendConstants
+
+    @property
+    def constants_model_class(self) -> Type[TrendConstants]:
+        """Return the constants model class used by this class."""
+        return TrendConstants
+
+    def __init__(self, recipe_run_id: int, workflow_name: str, workflow_version: str):
+        super().__init__(
+            recipe_run_id=recipe_run_id,
+            workflow_name=workflow_name,
+            workflow_version=workflow_version,
+        )
+        self.parameters = TrendParameters(
+            scratch=self.scratch,
+            obs_ip_start_time=self.constants.earliest_ip_start_time,
+            instrument=self.constants.instrument,
+        )
+
+    def run(self):
+        """Produce a single trend output file for each combination of arm ID, pac fit options, and instrument fit options."""
+        for arm_id in self.constants.arm_id_list:
+            # HDUs that are the same for all instrument/fit options
+            calibration_sequence_hdu = self.generate_calibration_sequence_hdu(arm_id=arm_id)
+
+            for fit_options in self.parameters.fit_options_list:
+                fit_info_hdu = self.generate_fit_info_hdu(fit_options)
+                for inst_options in self.parameters.instrument_processing_options:
+                    hdu_list = fits.HDUList([fits.PrimaryHDU()])
+
+                    hdu_list.append(
+                        self.generate_inventory_hdu(
+                            arm_id=arm_id, fit_options=fit_options, instrument_options=inst_options
+                        )
+                    )
+
+                    hdu_list.append(calibration_sequence_hdu)
+                    hdu_list.append(fit_info_hdu)
+
+                    common_args = {
+                        "arm_id": arm_id,
+                        "fit_options": fit_options,
+                        "instrument_options": inst_options,
+                    }
+                    hdu_list.append(self.generate_fit_parameters_hdu(**common_args))
+                    hdu_list.append(self.generate_modulation_matrices_hdu(**common_args))
+                    hdu_list.append(self.generate_best_fit_flux_hdu(**common_args))
+                    hdu_list.append(self.generate_fit_residuals_hdu(**common_args))
+                    if len(self.beam_list(arm_id)) == 2:
+                        hdu_list.append(self.generate_single_beam_DMPD_hdu(beam=1, **common_args))
+                        hdu_list.append(self.generate_single_beam_DMPD_hdu(beam=2, **common_args))
+                    hdu_list.append(self.generate_DMPD_hdu(**common_args))
+
+                    self.write_trend_output(hdu_list, arm_id, fit_options, inst_options)
+
+    @staticmethod
+    def beam_list(arm_id: str | int) -> list[int]:
+        """Return the list of beams the need to be iterated over for a given arm id."""
+        if arm_id == "CI":
+            return [1]
+        return [1, 2]
+
+    @staticmethod
+    def convert_float64_to_float32(input_array: np.ndarray) -> np.ndarray:
+        """
+        Convert an input `np.float64` array to `np.float32`.
+
+        Any other array type is returned unchanged.
+        """
+        if np.issubdtype(input_array.dtype, np.float64):
+            return input_array.astype(np.float32, casting="same_kind")
+        return input_array
+
+    def build_hdu(
+        self, header: fits.Header, data: np.ndarray | None = None
+    ) -> fits.ImageHDU | fits.CompImageHDU:
+        """
+        Construct an HDU from input header and data objects.
+
+        This method also formats the header to be nice and readable.
+
+        If ``data`` is not `None` then the returned HDU will be a `~astropy.io.fits.CompImageHDU`. Header-only HDUs are
+        returned as `~astropy.io.fits.ImageHDU` objects.
+        """
+        if data is None:
+            hdu_cls = fits.ImageHDU
+            extra_args = dict()
+        else:
+            data = self.convert_float64_to_float32(data)
+            hdu_cls = fits.CompImageHDU
+            extra_args = {"tile_shape": self.compute_tile_size_for_array(data)}
+
+        hdu = hdu_cls(header=header, data=data, **extra_args)
+        formatted_header = reformat_trend_header(hdu.header)
+        hdu = hdu_cls(header=formatted_header, data=data, **extra_args)
+
+        return hdu
+
+    def trend_filename(
+        self, arm_id: int | str, fit_options: FitOptions, instrument_options: InstrumentOptionsBase
+    ) -> None:
+        """
+        Construct a single Trend output file name.
+
+        The format is:
+
+        trend_polcal_DATE_INSTRUMENT_ARM_FITOPTION-INSTRUMENTOPTION_DATASETID.fits
+
+        Example
+        -------
+        "trend_polcal_2020-03-14T12_34_56_DLNIRSP_JBand_tpol_free-no_smooth_ADJFSI.fits"
+        """
+        instrument = self.constants.instrument
+        date_beg = (
+            self.constants.earliest_ip_start_time.replace("-", "_")
+            .replace(":", "_")
+            .replace(".", "_")
+        )
+        options_str = self.compute_full_fit_opt_id(
+            fit_options=fit_options, instrument_options=instrument_options
+        )
+        return f"trend_polcal_{date_beg}_{instrument}_{arm_id}_{options_str}_{self.constants.dataset_id}.fits"
+
+    @staticmethod
+    def compute_full_fit_opt_id(
+        fit_options: FitOptions, instrument_options: InstrumentOptionsBase
+    ) -> str:
+        """Concatenate the 'names' of both the pac-fit and instrument-processing options."""
+        return f"{fit_options.name}-{instrument_options.name}"
+
+    def update_framevol(self, relative_path: str) -> None:
+        """
+        Update the second HDU's (Inventory) FRAMEVOL key to be exactly the size of the file on-disk.
+
+        This is different than the L1 `update_framevol` method because we only update a single HDU (the Inventory HDU).
+        """
+        full_path = self.scratch.workflow_base_path / relative_path
+        compressed_size = full_path.stat().st_size / 1024 / 1024
+        hdul = fits.open(full_path, mode="update", disable_image_compression=True)
+        hdul[1].header["FRAMEVOL"] = compressed_size
+        hdul.flush()
+        del hdul
+
+    def write_trend_output(
+        self,
+        hdu_list: fits.HDUList,
+        arm_id: int | str,
+        fit_options: FitOptions,
+        instrument_options: InstrumentOptionsBase,
+    ) -> None:
+        """Write the given HDUList to scratch, tag with FRAME and OUTPUT, and update the FRAMEVOL header key."""
+        relative_path = self.trend_filename(
+            arm_id=arm_id, fit_options=fit_options, instrument_options=instrument_options
+        )
+        tags = [
+            TrendTag.output(),
+            TrendTag.frame(),
+            # These next tags are only for easing test writing and debugging
+            TrendTag.arm_id(arm_id),
+            TrendTag.pac_fit_options(fit_options.name),
+            TrendTag.instrument_processing_options(instrument_options.name),
+        ]
+
+        self.write(
+            data=hdu_list,
+            tags=tags,
+            encoder=fits_hdulist_encoder,
+            relative_path=relative_path,
+        )
+
+        self.update_framevol(relative_path)
+
+        # TODO: Should there be a different configuration variable for `validate_trend_on_write`?
+        if self.validate_l1_on_write:
+            # Replace this logic with the fits_header_decoder once FITS library keys have been removed from the SPEC
+            # TODO: Actually do something here once validator is ready
+            pass
+            # with fits.open(self.scratch.absolute_path(relative_path)) as hdul:
+            #     header = hdul[1].header
+            #     spec214_validator.validate(header, extra=False)
+
+    def generate_fit_info_hdu(self, fit_options: FitOptions) -> fits.ImageHDU:
+        """
+        Generate the data-less "Fit info" HDU.
+
+        Header values are pulled from the fit mode definitions that live in `dkist-processing-pac`.
+        """
+        switches, vary = PolcalDresserParameters.load_fit_mode(fit_mode=fit_options.fit_mode_name)
+
+        # If we use db M12 values then Q_in *never* varies. See `*-pac` for more info
+        q_in_free = vary["Q_in"] and not switches["use_M12"]
+
+        header = fits.Header()
+        header["EXTNAME"] = "Fit info"
+        header["QINFREE"] = q_in_free
+        header["TPOLFREE"] = vary["t_pol"]
+        header["TRETFREE"] = vary["t_ret"]
+        header["RMITREND"] = fit_options.remove_I_trend
+        header["ISYSPERS"] = switches["I_sys_per_CS_step"]
+        header["GLBLTRNS"] = switches["global_transmission"]
+        header["GLBLRET"] = switches["global_retardance"]
+
+        # These perturbation parameters are not currently used
+        header["X34PERT"] = 0
+        header["T34PERT"] = 0
+        header["X56PERT"] = 0
+        header["T56PERT"] = 0
+        header["UINPERT"] = 0
+        header["VINPERT"] = 0
+
+        return self.build_hdu(header=header)
+
+    def generate_inventory_hdu(
+        self, arm_id: str | int, fit_options: FitOptions, instrument_options: InstrumentOptionsBase
+    ) -> fits.ImageHDU:
+        """
+        Generate the data-less "Inventory" HDU.
+
+        This is also where instrument-specific headers are added.
+        """
+        header = fits.Header()
+        header["EXTNAME"] = "Inventory"
+        header["DSETID"] = self.constants.dataset_id
+        header["PROCTYPE"] = "POLCAL"
+        # This is just a placeholder value, but it's needed so FRAMEVOL gets properly commented and placed during header formatting
+        header["FRAMEVOL"] = -1.0
+        header["RRUNID"] = self.recipe_run_id
+        header["RECIPEID"] = self.metadata_store_recipe_run.recipeInstance.recipeId
+        header["RINSTID"] = self.metadata_store_recipe_run.recipeInstanceId
+        header["FILENAME"] = self.trend_filename(
+            arm_id=arm_id, fit_options=fit_options, instrument_options=instrument_options
+        )
+        header["HEADVERS"] = spec_version
+        header["HEAD_URL"] = f"{self.docs_base_url}/projects/data-products/en/v{spec_version}"
+        header["INFO_URL"] = self.docs_base_url
+        calvers = self.version_from_module_name()
+        header["CALVERS"] = calvers
+        header["CAL_URL"] = (
+            f"{self.docs_base_url}/projects/trend/en/v{calvers}/{self.workflow_name}.html"
+        )
+        header["WKFLNAME"] = self.workflow_name
+        header["WKFLVERS"] = self.workflow_version
+        header["MANPROCD"] = self.workflow_had_manual_intervention
+        header["FILE_ID"] = uuid.uuid4().hex
+        header["OBSPR_ID"] = self.constants.observing_program_id_list[0]
+        header["EXTOBSID"] = ",".join(self.constants.observing_program_id_list[1:])
+        header["EXPER_ID"] = self.constants.experiment_id_list[0]
+        header["EXTEXPID"] = ",".join(self.constants.experiment_id_list[1:])
+        header["PROP_ID"] = self.constants.proposal_id_list[0]
+        header["EXTPRPID"] = ",".join(self.constants.proposal_id_list[1:])
+        header["HLSVERS"] = self.constants.hls_version_list[0]
+        header["EXTHLSV"] = ",".join(self.constants.hls_version_list[1:])
+        header["OPTDATE"] = self.constants.earliest_ip_start_time
+        header["LINEWAV"] = self.constants.wavelength_for_arm[arm_id]
+        header["INSTRUME"] = self.constants.instrument
+        header["FITOPTID"] = self.compute_full_fit_opt_id(
+            fit_options=fit_options, instrument_options=instrument_options
+        )
+        header["ORIGIN"] = "National Solar Observatory"
+        header["TELESCOP"] = "Daniel K. Inouye Solar Telescope"
+        header["OBSRVTRY"] = "Haleakala High Altitude Observatory Site"
+        header["NETWORK"] = "NSF-DKIST"
+        header["DATE"] = Time.now().fits
+
+        match self.constants.instrument:
+            case InstrumentName.visp:
+                self.add_visp_headers(
+                    header=header, arm_id=arm_id, instrument_options=instrument_options
+                )
+            case InstrumentName.cryonirsp:
+                self.add_cryonirsp_headers(
+                    header=header, arm_id=arm_id, instrument_options=instrument_options
+                )
+            case InstrumentName.dlnirsp:
+                self.add_dlnirsp_headers(
+                    header=header, arm_id=arm_id, instrument_options=instrument_options
+                )
+
+        return self.build_hdu(header=header)
+
+    def add_visp_headers(
+        self, header: fits.Header, arm_id: int, instrument_options: VispInstrumentOptions
+    ) -> None:
+        """Add keys in the "visp" trend table to a header."""
+        header["VSPARMID"] = arm_id
+        header["VSPNUMST"] = self.constants.num_modstates
+        header["VSPGRTAN"] = self.constants.visp_grating_angle_for_arm[arm_id]
+        header["VSPARMPS"] = self.constants.visp_arm_position_for_arm[arm_id]
+        header["VSPGRTCN"] = self.constants.visp_grating_constant_for_arm[arm_id]
+        header["VSPNSPIX"] = instrument_options.num_spatial_px
+        header["VSPNWPIX"] = instrument_options.num_spectral_px
+        header["VSPSFORD"] = instrument_options.spatial_smoothing_fit_order
+
+    def add_cryonirsp_headers(
+        self, header: fits.Header, arm_id: str, instrument_options: CryonirspInstrumentOptions
+    ) -> None:
+        """Add keys in the "cryonirsp" trend table to a header."""
+        header["CNARMID"] = arm_id
+        header["CNMODNST"] = self.constants.num_modstates
+        header["CNFILTNP"] = self.constants.cryonirsp_od_filter_name_for_arm[arm_id]
+        header["CNGRTPOS"] = self.constants.cryonirsp_grating_angle_for_arm[arm_id]
+        header["CNGRTLAT"] = self.constants.cryonirsp_grating_littrow_angle_for_arm[arm_id]
+        header["CNGRTCON"] = self.constants.cryonirsp_grating_constant_for_arm[arm_id]
+        header["CNSLITW"] = self.constants.cryonirsp_slit_width_for_arm[arm_id]
+        header["CNNSPIX"] = instrument_options.num_spatial_px
+        header["CNNWPIX"] = instrument_options.num_spectral_px
+
+    def add_dlnirsp_headers(
+        self, header: fits.Header, arm_id: str, instrument_options: DlnirspInstrumentOptions
+    ) -> None:
+        """Add keys in the "dlnirsp" trend table to a header."""
+        header["DLARMID"] = arm_id
+        header["DLNUMST"] = self.constants.num_modstates
+        header["DLGRTAN"] = self.constants.dlnirsp_grating_angle_for_arm[arm_id]
+        header["DLARMPS"] = self.constants.dlnirsp_arm_position_for_arm[arm_id]
+        header["DLGRTCN"] = self.constants.dlnirsp_grating_constant_for_arm[arm_id]
+        header["DLNSPIX"] = instrument_options.num_spatial_px
+        header["DLNWPIX"] = instrument_options.num_spectral_px
+        header["DLSFORD"] = instrument_options.spatial_smoothing_fit_order
+
+    def generate_calibration_sequence_hdu(self, arm_id: str | int) -> fits.ImageHDU:
+        """Generate the "Calibration sequence" HDU that contains information about the full set of input GOS configurations."""
+        header = fits.Header()
+        header["EXTNAME"] = "Calibration sequence"
+        header["CSAX1"] = "CS_STEP"
+        header["CSAX2"] = "OPTIC"
+        header["CSAX3"] = "Calibration Sequence"
+        header["ONAME1"] = "POL_IN"
+        header["ONAME2"] = "POL_ANG_DEG"
+        header["ONAME3"] = "RET_IN"
+        header["ONAME4"] = "RET_ANG_DEG"
+        header["ONAME5"] = "DARK_IN"
+        header["ONAME6"] = "TAZIMUTH_DEG"
+        header["ONAME7"] = "ELEV_ANG_DEG"
+        header["ONAME8"] = "TTBLANGL_DEG"
+
+        data = next(
+            self.read(
+                tags=[
+                    TrendTag.intermediate(),
+                    TrendTag.arm_id(arm_id),
+                    TrendTag.task_calibration_sequence(),
+                ],
+                decoder=fits_array_decoder,
+                auto_squeeze=False,
+            )
+        )
+
+        return self.build_hdu(header=header, data=data)
+
+    def generate_fit_parameters_hdu(
+        self, arm_id: str | int, fit_options: FitOptions, instrument_options: InstrumentOptionsBase
+    ) -> fits.CompImageHDU:
+        """
+        Generate the "Fit parameters" HDU that contains the full set of best-fit parameters.
+
+        The actual array of fit parameters is constructed in `~dkist_processing_trend.tasks.run_pac_fitter.RunPacFitter.write_best_fit_parameters`.
+
+        This HDU is compressed because the FOV size can be large.
+        """
+        header = fits.Header()
+        header["EXTNAME"] = "Fit parameters"
+        header["NCSSTEPS"] = self.constants.num_cs_steps
+        header["FPAX1"] = "[value, init_value, free_in_fit]"
+        header["FPAX2"] = "Parameter"
+        header["FPAX3"] = "Calibration Sequence"
+        header["FPAX4"] = "Detector FITS axis 1"
+        header["FPAX5"] = "Detector FITS axis 2"
+        header["FPAX6"] = "Beam"
+        header["PNAME001"] = "X12"
+        header["PNAME002"] = "Tau 12"
+        header["PNAME003"] = "X34"
+        header["PNAME004"] = "Tau 34"
+        header["PNAME005"] = "X56"
+        header["PNAME006"] = "Tau 56"
+        header["PNAME007"] = "Q_in"
+        header["PNAME008"] = "U_in"
+        header["PNAME009"] = "V_in"
+        header["PNAME010"] = "t_pol"
+        header["PNAME011"] = "t_ret"
+        header["PNAME012"] = "ret_r"
+        header["PNAME013"] = "ret_45"
+        header["PNAME014"] = "ret_h"
+        header["PNAME015"] = "py"
+        for s in range(self.constants.num_cs_steps):
+            # TODO: Account for multiple polcal OPs by having "I_sys_CSXX_stepNN"
+            header[f"PNAME{s + 16:03d}"] = f"I_sys_step{s:03d}"
+
+        all_beam_data_list = []
+        for beam in self.beam_list(arm_id):
+            all_beam_data_list.append(
+                next(
+                    self.read(
+                        tags=[
+                            TrendTag.intermediate(),
+                            TrendTag.arm_id(arm_id),
+                            TrendTag.pac_fit_options(fit_options.name),
+                            TrendTag.instrument_processing_options(instrument_options.name),
+                            TrendTag.beam(beam),
+                            TrendTag.task_best_fit_parameters(),
+                        ],
+                        decoder=fits_array_decoder,
+                        auto_squeeze=False,
+                    )
+                )
+            )
+
+        # Will add the correct dimension even for a single beam
+        all_beam_data = np.stack(all_beam_data_list, axis=0)
+
+        return self.build_hdu(header=header, data=all_beam_data)
+
+    def generate_modulation_matrices_hdu(
+        self, arm_id: str | int, fit_options: FitOptions, instrument_options: InstrumentOptionsBase
+    ) -> fits.CompImageHDU:
+        """
+        Generate the "Modulation matrices" HDU that contains all modulation matrices for the given instrument FOV.
+
+        This is where we convert the DEmodulation matrices produced by `dkist_processing_pac` and
+        `~dkist_processing_trend.tasks.run_pac_fitter.RunPacFitter.write_best_fit_demodulation_matrices` into modulation matrices.
+
+        This HDU is compressed because the FOV size can be large.
+        """
+        header = fits.Header()
+        header["EXTNAME"] = "Modulation matrices"
+        header["MMAX1"] = "Stokes parameter"
+        header["MMAX2"] = "Modulator state"
+        header["MMAX3"] = "Detector FITS axis 1"
+        header["MMAX4"] = "Detector FITS axis 2"
+        header["MMAX5"] = "Beam"
+
+        all_beam_data_list = []
+        for beam in self.beam_list(arm_id):
+            beam_demod = next(
+                self.read(
+                    tags=[
+                        TrendTag.intermediate(),
+                        TrendTag.arm_id(arm_id),
+                        TrendTag.pac_fit_options(fit_options.name),
+                        TrendTag.instrument_processing_options(instrument_options.name),
+                        TrendTag.beam(beam),
+                        TrendTag.task_best_fit_demodulation_matrices(),
+                    ],
+                    decoder=fits_array_decoder,
+                    auto_squeeze=False,
+                )
+            )
+            fov_shape = beam_demod.shape[:-2]
+            num_points = np.prod(fov_shape)
+            beam_mod_mat = np.empty(fov_shape + (self.constants.num_modstates, 4))
+
+            for point in range(num_points):
+                idx = np.unravel_index(point, fov_shape)
+                point_demod = beam_demod[idx]
+                if np.sum(np.isnan(point_demod)) > 0:
+                    point_mod = np.full((self.constants.num_modstates, 4), np.nan)
+                else:
+                    point_mod = np.linalg.pinv(point_demod)
+
+                beam_mod_mat[idx] = point_mod
+
+            all_beam_data_list.append(beam_mod_mat)
+
+        # Will add the correct dimension even for a single beam
+        all_beam_data = np.stack(all_beam_data_list, axis=0)
+
+        return self.build_hdu(header=header, data=all_beam_data)
+
+    def generate_best_fit_flux_hdu(
+        self, arm_id: str | int, fit_options: FitOptions, instrument_options: InstrumentOptionsBase
+    ) -> fits.CompImageHDU:
+        """
+        Generate the "Best fit flux" HDU that contains the full set of best-fit model fluxes.
+
+        The actual flux array is constructed in `~dkist_processing_trend.tasks.run_pac_fitter.RunPacFitter.write_best_fit_flux_and_residuals`.
+
+        This HDU is compressed because the FOV size can be large.
+        """
+        header = fits.Header()
+        header["EXTNAME"] = "Best fit flux"
+        header["BFFAX1"] = "CS step"
+        header["BFFAX2"] = "Modulator state"
+        header["BFFAX3"] = "Detector FITS axis 1"
+        header["BFFAX4"] = "Detector FITS axis 2"
+        header["BFFAX5"] = "Beam"
+
+        all_beam_data_list = []
+        for beam in self.beam_list(arm_id):
+            all_beam_data_list.append(
+                next(
+                    self.read(
+                        tags=[
+                            TrendTag.intermediate(),
+                            TrendTag.arm_id(arm_id),
+                            TrendTag.pac_fit_options(fit_options.name),
+                            TrendTag.instrument_processing_options(instrument_options.name),
+                            TrendTag.beam(beam),
+                            TrendTag.task_best_fit_flux(),
+                        ],
+                        decoder=fits_array_decoder,
+                        auto_squeeze=False,
+                    )
+                )
+            )
+
+        # Will add the correct dimension even for a single beam
+        all_beam_data = np.stack(all_beam_data_list, axis=0)
+
+        return self.build_hdu(header=header, data=all_beam_data)
+
+    def generate_fit_residuals_hdu(
+        self, arm_id: str | int, fit_options: FitOptions, instrument_options: InstrumentOptionsBase
+    ) -> fits.CompImageHDU:
+        """
+        Generate the "Best fit residuals" HDU that contains the fitting residuals of the best-fit solution.
+
+        The actual flux array is constructed in `~dkist_processing_trend.tasks.run_pac_fitter.RunPacFitter.write_best_fit_flux_and_residuals`.
+
+        This HDU is compressed because the FOV size can be large.
+        """
+        header = fits.Header()
+        header["EXTNAME"] = "Best fit residuals"
+        header["BFRAX1"] = "CS step"
+        header["BFRAX2"] = "Modulator state"
+        header["BFRAX3"] = "Detector FITS axis 1"
+        header["BFRAX4"] = "Detector FITS axis 2"
+        header["BFRAX5"] = "Beam"
+
+        all_beam_data_list = []
+        for beam in self.beam_list(arm_id):
+            all_beam_data_list.append(
+                next(
+                    self.read(
+                        tags=[
+                            TrendTag.intermediate(),
+                            TrendTag.arm_id(arm_id),
+                            TrendTag.pac_fit_options(fit_options.name),
+                            TrendTag.instrument_processing_options(instrument_options.name),
+                            TrendTag.beam(beam),
+                            TrendTag.task_fit_residuals(),
+                        ],
+                        decoder=fits_array_decoder,
+                        auto_squeeze=False,
+                    )
+                )
+            )
+
+        # Will add the correct dimension even for a single beam
+        all_beam_data = np.stack(all_beam_data_list, axis=0)
+
+        return self.build_hdu(header=header, data=all_beam_data)
+
+    def generate_single_beam_DMPD_hdu(
+        self,
+        beam: int,
+        arm_id: str | int,
+        fit_options: FitOptions,
+        instrument_options: InstrumentOptionsBase,
+    ) -> fits.CompImageHDU:
+        """
+        Generate the "Demodulated polcal data" HDU that contains the "processed" and demodulated input data for a single beam.
+
+        This HDU is compressed because this HDU is huge.
+        """
+        header = fits.Header()
+        header["EXTNAME"] = f"Beam {beam} demodulated polcal data"
+        header["DMPDAX1"] = "Input data axis 1"
+        header["DMPDAX2"] = "Input data axis 2"
+        header["DMPDAX3"] = "Stokes"
+        header["DMPDAX4"] = "CS Step"
+        header["DMPDAX5"] = "Calibration Sequence"
+
+        data = next(
+            self.read(
+                tags=[
+                    TrendTag.intermediate(),
+                    TrendTag.arm_id(arm_id),
+                    TrendTag.beam(beam),
+                    TrendTag.pac_fit_options(fit_options.name),
+                    TrendTag.instrument_processing_options(instrument_options.name),
+                    TrendTag.task_single_beam_demodulated_polcal_data(),
+                ],
+                decoder=fits_array_decoder,
+                auto_squeeze=False,
+            )
+        )
+
+        return self.build_hdu(header=header, data=data)
+
+    def generate_DMPD_hdu(
+        self, arm_id: str | int, fit_options: FitOptions, instrument_options: InstrumentOptionsBase
+    ) -> fits.CompImageHDU:
+        """
+        Generate the "Demodulated polcal data" HDU that contains the "processed" and demodulated input data.
+
+        This HDU is compressed because this HDU is huge.
+        """
+        header = fits.Header()
+        header["EXTNAME"] = "Demodulated polcal data"
+        header["DMPDAX1"] = "Input data axis 1"
+        header["DMPDAX2"] = "Input data axis 2"
+        header["DMPDAX3"] = "Stokes"
+        header["DMPDAX4"] = "CS Step"
+        header["DMPDAX5"] = "Calibration Sequence"
+
+        data = next(
+            self.read(
+                tags=[
+                    TrendTag.intermediate(),
+                    TrendTag.arm_id(arm_id),
+                    TrendTag.pac_fit_options(fit_options.name),
+                    TrendTag.instrument_processing_options(instrument_options.name),
+                    TrendTag.task_demodulated_polcal_data(),
+                ],
+                decoder=fits_array_decoder,
+                auto_squeeze=False,
+            )
+        )
+
+        return self.build_hdu(header=header, data=data)

dkist_processing_trend/tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for testing."""