dkist-processing-common 11.7.1rc1__py3-none-any.whl → 11.8.0rc1__py3-none-any.whl

dkist_processing_common/tasks/parse_l0_input_data.py

@@ -34,16 +34,23 @@ from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.flower_pot import Thorn
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.models.task_name import TaskName
+from dkist_processing_common.parsers.average_bud import TaskAverageBud
 from dkist_processing_common.parsers.experiment_id_bud import ContributingExperimentIdsBud
 from dkist_processing_common.parsers.experiment_id_bud import ExperimentIdBud
+from dkist_processing_common.parsers.observing_program_id_bud import (
+    TaskContributingObservingProgramExecutionIdsBud,
+)
 from dkist_processing_common.parsers.proposal_id_bud import ContributingProposalIdsBud
 from dkist_processing_common.parsers.proposal_id_bud import ProposalIdBud
+from dkist_processing_common.parsers.task import parse_header_ip_task_with_gains
 from dkist_processing_common.parsers.time import AverageCadenceBud
 from dkist_processing_common.parsers.time import MaximumCadenceBud
 from dkist_processing_common.parsers.time import MinimumCadenceBud
+from dkist_processing_common.parsers.time import TaskDateBeginBud
 from dkist_processing_common.parsers.time import TaskExposureTimesBud
 from dkist_processing_common.parsers.time import TaskReadoutExpTimesBud
 from dkist_processing_common.parsers.time import VarianceCadenceBud
+from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
 from dkist_processing_common.parsers.unique_bud import UniqueBud
 from dkist_processing_common.tasks.base import WorkflowTaskBase
 
@@ -59,9 +66,247 @@ logger = logging.getLogger(__name__)
 S = TypeVar("S", bound=Stem)
 
 
+def dataset_extra_bud_factory() -> list[S]:
+    """Provide constant buds for use in dataset extras."""
+    return [
+        UniqueBud(constant_name=BudName.camera_id, metadata_key=MetadataKey.camera_id),
+        UniqueBud(constant_name=BudName.camera_name, metadata_key=MetadataKey.camera_name),
+        UniqueBud(
+            constant_name=BudName.camera_bit_depth, metadata_key=MetadataKey.camera_bit_depth
+        ),
+        UniqueBud(
+            constant_name=BudName.hardware_binning_x, metadata_key=MetadataKey.hardware_binning_x
+        ),
+        UniqueBud(
+            constant_name=BudName.hardware_binning_y, metadata_key=MetadataKey.hardware_binning_x
+        ),
+        UniqueBud(
+            constant_name=BudName.software_binning_x, metadata_key=MetadataKey.software_binning_x
+        ),
+        UniqueBud(
+            constant_name=BudName.software_binning_y, metadata_key=MetadataKey.software_binning_y
+        ),
+        UniqueBud(
+            constant_name=BudName.hls_version,
+            metadata_key=MetadataKey.hls_version,
+        ),
+        TaskContributingObservingProgramExecutionIdsBud(
+            constant_name=BudName.dark_observing_program_execution_ids,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskContributingObservingProgramExecutionIdsBud(
+            constant_name=BudName.solar_gain_observing_program_execution_ids,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskContributingObservingProgramExecutionIdsBud(
+            constant_name=BudName.polcal_observing_program_execution_ids,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_num_raw_frames_per_fpa,
+            metadata_key=MetadataKey.num_raw_frames_per_fpa,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_num_raw_frames_per_fpa,
+            metadata_key=MetadataKey.num_raw_frames_per_fpa,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_telescope_tracking_mode,
+            metadata_key=MetadataKey.telescope_tracking_mode,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_telescope_tracking_mode,
+            metadata_key=MetadataKey.telescope_tracking_mode,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_coude_table_tracking_mode,
+            metadata_key=MetadataKey.coude_table_tracking_mode,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_coude_table_tracking_mode,
+            metadata_key=MetadataKey.coude_table_tracking_mode,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_telescope_scanning_mode,
+            metadata_key=MetadataKey.telescope_scanning_mode,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_telescope_scanning_mode,
+            metadata_key=MetadataKey.telescope_scanning_mode,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_level3_status,
+            metadata_key=MetadataKey.gos_level3_status,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_level3_status,
+            metadata_key=MetadataKey.gos_level3_status,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_level3_lamp_status,
+            metadata_key=MetadataKey.gos_level3_lamp_status,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_level3_lamp_status,
+            metadata_key=MetadataKey.gos_level3_lamp_status,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_polarizer_status,
+            metadata_key=MetadataKey.gos_polarizer_status,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_polarizer_status,
+            metadata_key=MetadataKey.gos_polarizer_status,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_polarizer_angle,
+            metadata_key=MetadataKey.gos_polarizer_angle,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_polarizer_angle,
+            metadata_key=MetadataKey.gos_polarizer_angle,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_retarder_status,
+            metadata_key=MetadataKey.gos_retarder_status,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_retarder_status,
+            metadata_key=MetadataKey.gos_retarder_status,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_retarder_angle,
+            metadata_key=MetadataKey.gos_retarder_angle,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_retarder_angle,
+            metadata_key=MetadataKey.gos_retarder_angle,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_level0_status,
+            metadata_key=MetadataKey.gos_level0_status,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_level0_status,
+            metadata_key=MetadataKey.gos_level0_status,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.dark_average_light_level,
+            metadata_key=MetadataKey.light_level,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.solar_gain_average_light_level,
+            metadata_key=MetadataKey.light_level,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.polcal_average_light_level,
+            metadata_key=MetadataKey.light_level,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.dark_average_telescope_elevation,
+            metadata_key=MetadataKey.elevation,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.solar_gain_average_telescope_elevation,
+            metadata_key=MetadataKey.elevation,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.polcal_average_telescope_elevation,
+            metadata_key=MetadataKey.elevation,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.dark_average_coude_table_angle,
+            metadata_key=MetadataKey.table_angle,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.solar_gain_average_coude_table_angle,
+            metadata_key=MetadataKey.table_angle,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.polcal_average_coude_table_angle,
+            metadata_key=MetadataKey.table_angle,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.dark_average_telescope_azimuth,
+            metadata_key=MetadataKey.azimuth,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.solar_gain_average_telescope_azimuth,
+            metadata_key=MetadataKey.azimuth,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.polcal_average_telescope_azimuth,
+            metadata_key=MetadataKey.azimuth,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskDateBeginBud(
+            constant_name=BudName.dark_date_begin,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskDateBeginBud(
+            constant_name=BudName.solar_gain_date_begin,
+            ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
+        ),
+        TaskDateBeginBud(
+            constant_name=BudName.polcal_date_begin,
+            ip_task_types=TaskName.polcal,
+        ),
+    ]
+
+
 def default_constant_bud_factory() -> list[S]:
     """Provide default constant buds for use in common parsing tasks."""
-    return [
+    return dataset_extra_bud_factory() + [
         UniqueBud(constant_name=BudName.instrument, metadata_key=MetadataKey.instrument),
         ProposalIdBud(),
         ContributingProposalIdsBud(),

dkist_processing_common/tasks/trial_catalog.py

@@ -5,15 +5,18 @@ import logging
 from datetime import datetime
 from itertools import chain
 from pathlib import Path
+from typing import Any
 from typing import Generator
 from uuid import uuid4
 
 from dkist_processing_common.codecs.asdf import asdf_fileobj_encoder
+from dkist_processing_common.codecs.basemodel import basemodel_decoder
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.json import json_encoder
 from dkist_processing_common.codecs.path import path_decoder
 from dkist_processing_common.codecs.quality import quality_data_decoder
 from dkist_processing_common.models.fits_access import FitsAccessBase
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.output_data_base import OutputDataBase
 
@@ -138,11 +141,15 @@ class CreateTrialAsdf(OutputDataBase):
 
     def run(self) -> None:
         """Generate an ASDF file simulating the ASDF file that would be produced when cataloging the dataset."""
+        with self.telemetry_span("Collate input dataset parameters"):
+            parameters = self.parse_input_dataset_parameters()
+
         with self.telemetry_span("Generate ASDF tree"):
             tree = asdf_tree_from_filenames(
                 filenames=self.absolute_output_frame_paths,
                 hdu=1,  # compressed
                 relative_to=self.scratch.workflow_base_path,
+                parameters=parameters,
             )
 
         trial_history = [
@@ -169,6 +176,37 @@ class CreateTrialAsdf(OutputDataBase):
                     ),
                 )
 
+    def parse_input_dataset_parameters(self) -> list[dict[str, Any]]:
+        """
+        Return the parameters associated with the dataset.
+
+        Returns
+        -------
+        list[dict[str, Any]]
+            A list of dictionaries, each containing a parameter name and its values.
+
+        Raises
+        ------
+        ValueError
+            If there is not exactly one ``InputDatasetPartDocumentList`` found.
+        """
+        part_docs_iter = self.read(
+            tags=Tag.input_dataset_parameters(),
+            decoder=basemodel_decoder,
+            model=InputDatasetPartDocumentList,
+        )
+        docs = list(part_docs_iter)
+
+        if not docs:
+            logger.warning("No parameter list decoded from files")
+            return []
+
+        if len(docs) > 1:
+            raise ValueError(f"Expected 1 parameter list, found {len(docs)}")
+
+        parameters = docs[0].model_dump(by_alias=True).get("doc_list", [])
+        return parameters
+
 
 class CreateTrialQualityReport(OutputDataBase):
     """

dkist_processing_common/tests/mock_metadata_store.py

@@ -6,6 +6,7 @@ import json
 from abc import ABC
 from abc import abstractmethod
 from datetime import datetime
+from datetime import timedelta
 from pathlib import Path
 from uuid import uuid4
 
@@ -234,3 +235,41 @@ def fake_gql_client():
     Convenience fixture for default mock GQL client. To customize, use fake_gql_client_factory.
     """
     return fake_gql_client_factory()
+
+
+def input_dataset_parameters_part_factory(
+    parameter_count: int = 1,
+    parameter_value_count: int = 1,
+    has_date: bool = False,
+    has_file: bool = False,
+) -> list[dict]:
+    """Create a mock InputDatasetPartDocumentList with parameters."""
+    result = [
+        {
+            "parameterName": uuid4().hex[:6],
+            "parameterValues": [
+                {"parameterValueId": i, "parameterValue": json.dumps(uuid4().hex)}
+                for i in range(parameter_value_count)
+            ],
+        }
+        for _ in range(parameter_count)
+    ]
+    if has_date:
+        base = datetime(2018, 9, 14, 0, 0, 0)  # This date is before any possible start dates
+        for parameter_index, data in enumerate(result):
+            for item in data["parameterValues"]:
+                dt = base + timedelta(days=parameter_index)
+                item["parameterValueStartDate"] = dt.isoformat()
+    if has_file:
+        for data in result:
+            param_list = data["parameterValues"]
+            for item in param_list:
+                item["parameterValue"] = json.dumps(
+                    {
+                        "__file__": {
+                            "bucket": "data",
+                            "objectKey": f"parameters/{data['parameterName']}/{uuid4().hex}.dat",
+                        }
+                    }
+                )
+    return result

dkist_processing_common/tests/test_fits_access.py

@@ -4,7 +4,6 @@ import numpy as np
 import pytest
 from astropy.io import fits
 
-from dkist_processing_common.models.fits_access import NOT_FOUND_MESSAGE
 from dkist_processing_common.models.fits_access import FitsAccessBase
 from dkist_processing_common.models.fits_access import MetadataKey
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
@@ -99,30 +98,16 @@ def hdu_with_no_data(complete_common_header):
 @pytest.fixture()
 def hdu_with_incomplete_common_header(complete_common_header):
     """
-    An HDU with data and a header missing one of the expected common by-frame keywords
+    An HDU with data and a header missing two of the expected common by-frame keywords
     """
     incomplete_header = complete_common_header
-    incomplete_header.pop("ELEV_ANG")
-    incomplete_header.pop("TAZIMUTH")
+    incomplete_header.pop(MetadataKey.elevation)
+    incomplete_header.pop(MetadataKey.azimuth)
     data = np.arange(9).reshape(3, 3)
     hdu = fits.PrimaryHDU(data, header=incomplete_header)
     return hdu
 
 
-class MetadataKeyWithOptionalKeys(StrEnum):
-    optional1 = "ELEV_ANG"
-    optional2 = "TAZIMUTH"
-
-
-class FitsAccessWithOptionalKeys(FitsAccessBase):
-    def __init__(self, hdu, name):
-        super().__init__(hdu, name)
-        self._set_metadata_key_value(MetadataKeyWithOptionalKeys.optional1, optional=True)
-        self._set_metadata_key_value(
-            MetadataKeyWithOptionalKeys.optional2, optional=True, default="SO_RAD"
-        )
-
-
 @pytest.fixture()
 def fits_file_path(tmp_path, complete_common_header):
     file_path = tmp_path / "foo.fits"
@@ -155,35 +140,36 @@ class MetadataKeyWithNaxisKeys(StrEnum):
 class FitsAccessWithNaxisKeys(FitsAccessBase):
     def __init__(self, hdu, name):
         super().__init__(hdu, name)
-        self._set_metadata_key_value(MetadataKeyWithNaxisKeys.naxis)
-        self._set_metadata_key_value(MetadataKeyWithNaxisKeys.naxis1)
-        self._set_metadata_key_value(MetadataKeyWithNaxisKeys.naxis2)
+        self.naxis = self.header[MetadataKeyWithNaxisKeys.naxis]
+        self.naxis1 = self.header[MetadataKeyWithNaxisKeys.naxis1]
+        self.naxis2 = self.header[MetadataKeyWithNaxisKeys.naxis2]
 
 
 def test_metadata_keys_in_access_bases(hdu_with_no_data):
     """
     Given: a set of metadata key names in the MetadataKey sting enumeration
     When: the FITS access classes define a set of attributes/properties
-    Then: the sets are the same
+    Then: the sets are the same and the values of the attributes/properties are correct
     """
     all_metadata_key_names = {mk.name for mk in MetadataKey}
     hdu = hdu_with_no_data
-    l1_properties = {k for k, v in L1FitsAccess.__dict__.items() if isinstance(v, property)}
-    l0_properties = {k for k, v in L0FitsAccess.__dict__.items() if isinstance(v, property)}
-    all_fits_access_properties = l1_properties | l0_properties
-    l1_dict_keys = {k for k, v in L1FitsAccess(hdu=hdu).__dict__.items()}
-    l0_dict_keys = {k for k, v in L0FitsAccess(hdu=hdu).__dict__.items()}
-    all_fits_access_dict_keys = l1_dict_keys | l0_dict_keys
-    instance_dict_keys = {"_hdu", "name", "auto_squeeze"}
-    keys_defined_in_fits_access = all_fits_access_dict_keys | all_fits_access_properties
-    assert all_metadata_key_names == keys_defined_in_fits_access - instance_dict_keys
+    fits_obj = L0FitsAccess(hdu=hdu)
+    fits_access_defined_attributes = {
+        k for k, v in fits_obj.__dict__.items() if k not in ["_hdu", "name", "auto_squeeze"]
+    }
+    l0_access_properties = {k for k, v in L0FitsAccess.__dict__.items() if isinstance(v, property)}
+    l1_access_properties = {k for k, v in L1FitsAccess.__dict__.items() if isinstance(v, property)}
+    fits_access_properties = l0_access_properties | l1_access_properties
+    assert fits_access_defined_attributes | fits_access_properties == all_metadata_key_names
+    for key in fits_access_defined_attributes:
+        assert getattr(fits_obj, key) == fits_obj.header[MetadataKey[key]]
 
 
 def test_from_single_hdu(hdu_with_complete_common_header):
     """
     Given: an HDU with expected, common by-frame keywords
     When: loading the HDU with the L0FitsAccess class
-    Then: all values for common keywords are exposed as properties on the fits_obj class
+    Then: values for common keywords are exposed as properties on the fits_obj class
     """
     fits_obj = L0FitsAccess(hdu_with_complete_common_header)
     assert fits_obj.elevation == 6.28
@@ -198,7 +184,7 @@ def test_l1_only_fits_access(hdu_with_complete_l1_only_header):
     """
     Given: an HDU with 214 L1-only headers
     When: loading the HDU with the L1FitsAccess class
-    Then: no errors are raised and all values are exposed
+    Then: no errors are raised and values are exposed
     """
     fits_obj = L1FitsAccess(hdu_with_complete_l1_only_header)
     assert fits_obj.elevation == 6.28
@@ -255,17 +241,6 @@ def test_no_header_value(hdu_with_incomplete_common_header):
         _ = L0FitsAccess(hdu_with_incomplete_common_header)
 
 
-def test_default_header_values(hdu_with_incomplete_common_header):
-    """
-    Given: an HDU with a header with missing common by-frame keywords
-    When: processing the HDU with a FITS access class that sets these keywords as optional
-    Then: the correct default values are set
-    """
-    fits_obj = FitsAccessWithOptionalKeys(hdu_with_incomplete_common_header, name="foo")
-    assert fits_obj.optional1 == MetadataKeyWithOptionalKeys.optional1 + NOT_FOUND_MESSAGE
-    assert fits_obj.optional2 == "SO_RAD"
-
-
 def test_as_subclass(hdu_with_complete_common_header):
     """
     Given: an instrument-specific fits_obj class that subclasses L0FitsAccess

dkist_processing_common/tests/test_input_dataset.py

@@ -1,5 +1,4 @@
 import json
-from datetime import datetime
 from typing import Any
 from uuid import uuid4
 
@@ -8,6 +7,7 @@ import pytest
 from dkist_processing_common.codecs.basemodel import basemodel_decoder
 from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.tests.mock_metadata_store import input_dataset_parameters_part_factory
 
 
 def input_dataset_frames_part_factory(bucket_count: int = 1) -> list[dict]:
@@ -25,42 +25,6 @@ def flatten_frame_parts(frame_parts: list[dict]) -> list[tuple[str, str]]:
     return result
 
 
-def input_dataset_parameters_part_factory(
-    parameter_count: int = 1,
-    parameter_value_count: int = 1,
-    has_date: bool = False,
-    has_file: bool = False,
-) -> list[dict]:
-    result = [
-        {
-            "parameterName": uuid4().hex[:6],
-            "parameterValues": [
-                {"parameterValueId": i, "parameterValue": json.dumps(uuid4().hex)}
-                for i in range(parameter_value_count)
-            ],
-        }
-        for _ in range(parameter_count)
-    ]
-    if has_date:
-        for data in result:
-            param_list = data["parameterValues"]
-            for item in param_list:
-                item["parameterValueStartDate"] = datetime(2022, 9, 14).isoformat()
-    if has_file:
-        for data in result:
-            param_list = data["parameterValues"]
-            for item in param_list:
-                item["parameterValue"] = json.dumps(
-                    {
-                        "__file__": {
-                            "bucket": "data",
-                            "objectKey": f"parameters/{data['parameterName']}/{uuid4().hex}.dat",
-                        }
-                    }
-                )
-    return result
-
-
 @pytest.mark.parametrize(
     "input_dataset_parts",
     [

dkist_processing_common/tests/test_parse_l0_input_data.py

@@ -97,9 +97,9 @@ class ViSPMetadataKey(StrEnum):
 class ViSPFitsAccess(FitsAccessBase):
     def __init__(self, hdu, name, auto_squeeze=False):
         super().__init__(hdu, name, auto_squeeze=auto_squeeze)
-        self._set_metadata_key_value(ViSPMetadataKey.num_mod)
-        self._set_metadata_key_value(ViSPMetadataKey.modstate)
-        self._set_metadata_key_value(ViSPMetadataKey.ip_task_type)
+        self.num_mod: int = self.header[ViSPMetadataKey.num_mod]
+        self.modstate: int = self.header[ViSPMetadataKey.modstate]
+        self.ip_task_type: str = self.header[ViSPMetadataKey.ip_task_type]
         self.name = name
 
 
@@ -260,18 +260,58 @@ def test_subclass_flowers(visp_parse_inputs_task, max_cs_step_time_sec):
     tag_pot, constant_pot = visp_parse_inputs_task.make_flower_pots()
 
     assert len(tag_pot.stems) == 1
-    assert len(constant_pot.stems) == 12
+    assert len(constant_pot.stems) == 60
     all_flower_names = [StemName.modstate]
     assert sorted([f.stem_name for f in tag_pot.stems]) == sorted(all_flower_names)
     all_bud_names = [b.stem_name for b in default_constant_bud_factory()] + [BudName.num_modstates]
     assert sorted([f.stem_name for f in constant_pot.stems]) == sorted(all_bud_names)
 
 
+def test_dataset_extra_bud_factory(visp_parse_inputs_task, max_cs_step_time_sec):
+    """
+    Given: ParseInputData child class with custom stems
+    When: Making the constant pot
+    Then: The multi-task dataset extra buds are created
+    """
+    _, constant_pot = visp_parse_inputs_task.make_flower_pots()
+    stem_names = [f.stem_name.value for f in constant_pot.stems]
+    bud_name_base = [
+        "DATE_BEGIN",
+        "OBSERVING_PROGRAM_EXECUTION_IDS",
+        "NUM_RAW_FRAMES_PER_FPA",
+        "TELESCOPE_TRACKING_MODE",
+        "COUDE_TABLE_TRACKING_MODE",
+        "TELESCOPE_SCANNING_MODE",
+        "AVERAGE_LIGHT_LEVEL",
+        "AVERAGE_TELESCOPE_ELEVATION",
+        "AVERAGE_COUDE_TABLE_ANGLE",
+        "AVERAGE_TELESCOPE_AZIMUTH",
+        "GOS_LEVEL3_STATUS",
+        "GOS_LEVEL3_LAMP_STATUS",
+        "GOS_POLARIZER_STATUS",
+        "GOS_POLARIZER_ANGLE",
+        "GOS_RETARDER_STATUS",
+        "GOS_RETARDER_ANGLE",
+        "GOS_LEVEL0_STATUS",
+    ]
+    for base in bud_name_base:
+        assert "SOLAR_GAIN_" + base in stem_names
+        if base not in [
+            "NUM_RAW_FRAMES_PER_FPA",
+            "TELESCOPE_TRACKING_MODE",
+            "COUDE_TABLE_TRACKING_MODE",
+            "TELESCOPE_SCANNING_MODE",
+        ]:
+            assert "DARK_" + base in stem_names
+        if "GOS" not in base:
+            assert "POLCAL_" + base in stem_names
+
+
 def test_constants_correct(parse_inputs_task):
     """
     Given: ParseInputData task with a populated constant FlowerPot
     When: Updating pipeline constants
-    Then: Pipeline constants are correctly populated
+    Then: A pipeline constant is correctly populated
     """
     _, constant_pot = parse_inputs_task.make_flower_pots()
     parse_inputs_task.update_constants(constant_pot)

dkist_processing_common/tests/test_quality_mixin.py

@@ -1214,20 +1214,12 @@ def wavecal_weights(wavecal_input_wavelength) -> np.ndarray:
 
 
 @pytest.fixture(scope="session")
-def wavecal_fit_result(wavecal_input_wavelength, wavecal_input_spectrum) -> FitResult:
+def wavecal_fit_result(wavecal_input_wavelength) -> FitResult:
     wavelength_params = WavelengthParameters(
         crpix=1, crval=10.0, dispersion=1, grating_constant=1, order=1, incident_light_angle=0
     )
-
-    residuals = np.random.random(wavecal_input_wavelength.size)
-    residuals[-1] = np.nan
-    minimizer_result = MinimizerResult(residual=residuals)
-    return FitResult(
-        wavelength_parameters=wavelength_params,
-        minimizer_result=minimizer_result,
-        input_wavelength_vector=wavecal_input_wavelength,
-        input_spectrum=wavecal_input_spectrum,
-    )
+    minimizer_result = MinimizerResult(residual=np.random.random(wavecal_input_wavelength.size))
+    return FitResult(wavelength_parameters=wavelength_params, minimizer_result=minimizer_result)
 
 
 @pytest.mark.parametrize(