dkist-processing-common 11.7.0rc2__py3-none-any.whl → 11.7.1rc1__py3-none-any.whl

dkist_processing_common/tasks/parse_l0_input_data.py

@@ -34,20 +34,16 @@ from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.flower_pot import Thorn
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.models.task_name import TaskName
-from dkist_processing_common.parsers.average_bud import TaskAverageBud
 from dkist_processing_common.parsers.experiment_id_bud import ContributingExperimentIdsBud
 from dkist_processing_common.parsers.experiment_id_bud import ExperimentIdBud
-from dkist_processing_common.parsers.id_bud import TaskContributingIdsBud
 from dkist_processing_common.parsers.proposal_id_bud import ContributingProposalIdsBud
 from dkist_processing_common.parsers.proposal_id_bud import ProposalIdBud
 from dkist_processing_common.parsers.time import AverageCadenceBud
 from dkist_processing_common.parsers.time import MaximumCadenceBud
 from dkist_processing_common.parsers.time import MinimumCadenceBud
-from dkist_processing_common.parsers.time import TaskDateBeginBud
 from dkist_processing_common.parsers.time import TaskExposureTimesBud
 from dkist_processing_common.parsers.time import TaskReadoutExpTimesBud
 from dkist_processing_common.parsers.time import VarianceCadenceBud
-from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
 from dkist_processing_common.parsers.unique_bud import UniqueBud
 from dkist_processing_common.tasks.base import WorkflowTaskBase
 
@@ -63,253 +59,9 @@ logger = logging.getLogger(__name__)
 S = TypeVar("S", bound=Stem)
 
 
-def dataset_extra_bud_factory() -> list[S]:
-    """Provide constant buds for use in dataset extras."""
-    return [
-        UniqueBud(constant_name=BudName.camera_id, metadata_key=MetadataKey.camera_id),
-        UniqueBud(constant_name=BudName.camera_name, metadata_key=MetadataKey.camera_name),
-        UniqueBud(
-            constant_name=BudName.camera_bit_depth, metadata_key=MetadataKey.camera_bit_depth
-        ),
-        UniqueBud(
-            constant_name=BudName.hardware_binning_x, metadata_key=MetadataKey.hardware_binning_x
-        ),
-        UniqueBud(
-            constant_name=BudName.hardware_binning_y, metadata_key=MetadataKey.hardware_binning_x
-        ),
-        UniqueBud(
-            constant_name=BudName.software_binning_x, metadata_key=MetadataKey.software_binning_x
-        ),
-        UniqueBud(
-            constant_name=BudName.software_binning_y, metadata_key=MetadataKey.software_binning_y
-        ),
-        UniqueBud(
-            constant_name=BudName.hls_version,
-            metadata_key=MetadataKey.hls_version,
-        ),
-        TaskContributingIdsBud(
-            constant_name=BudName.dark_observing_program_execution_id,
-            metadata_key=MetadataKey.observing_program_execution_id,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskContributingIdsBud(
-            constant_name=BudName.solar_gain_observing_program_execution_id,
-            metadata_key=MetadataKey.observing_program_execution_id,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskContributingIdsBud(
-            constant_name=BudName.polcal_observing_program_execution_id,
-            metadata_key=MetadataKey.observing_program_execution_id,
-            ip_task_types=TaskName.polcal,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.dark_num_raw_frames_per_fpa,
-            metadata_key=MetadataKey.num_raw_frames_per_fpa,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.solar_gain_num_raw_frames_per_fpa,
-            metadata_key=MetadataKey.num_raw_frames_per_fpa,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.polcal_num_raw_frames_per_fpa,
-            metadata_key=MetadataKey.num_raw_frames_per_fpa,
-            ip_task_types=TaskName.polcal,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.dark_telescope_tracking_mode,
-            metadata_key=MetadataKey.telescope_tracking_mode,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.solar_gain_telescope_tracking_mode,
-            metadata_key=MetadataKey.telescope_tracking_mode,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.polcal_telescope_tracking_mode,
-            metadata_key=MetadataKey.telescope_tracking_mode,
-            ip_task_types=TaskName.polcal,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.dark_coude_table_tracking_mode,
-            metadata_key=MetadataKey.coude_table_tracking_mode,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.solar_gain_coude_table_tracking_mode,
-            metadata_key=MetadataKey.coude_table_tracking_mode,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.polcal_coude_table_tracking_mode,
-            metadata_key=MetadataKey.coude_table_tracking_mode,
-            ip_task_types=TaskName.polcal,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.dark_telescope_scanning_mode,
-            metadata_key=MetadataKey.telescope_scanning_mode,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.solar_gain_telescope_scanning_mode,
-            metadata_key=MetadataKey.telescope_scanning_mode,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.polcal_telescope_scanning_mode,
-            metadata_key=MetadataKey.telescope_scanning_mode,
-            ip_task_types=TaskName.polcal,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.dark_gos_level3_status,
-            metadata_key=MetadataKey.gos_level3_status,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.solar_gain_gos_level3_status,
-            metadata_key=MetadataKey.gos_level3_status,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.dark_gos_level3_lamp_status,
-            metadata_key=MetadataKey.gos_level3_lamp_status,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.solar_gain_gos_level3_lamp_status,
-            metadata_key=MetadataKey.gos_level3_lamp_status,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.dark_gos_polarizer_status,
-            metadata_key=MetadataKey.gos_polarizer_status,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.solar_gain_gos_polarizer_status,
-            metadata_key=MetadataKey.gos_polarizer_status,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.dark_gos_polarizer_angle,
-            metadata_key=MetadataKey.gos_polarizer_angle,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.solar_gain_gos_polarizer_angle,
-            metadata_key=MetadataKey.gos_polarizer_angle,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.dark_gos_retarder_status,
-            metadata_key=MetadataKey.gos_retarder_status,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.solar_gain_gos_retarder_status,
-            metadata_key=MetadataKey.gos_retarder_status,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.dark_gos_retarder_angle,
-            metadata_key=MetadataKey.gos_retarder_angle,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.solar_gain_gos_retarder_angle,
-            metadata_key=MetadataKey.gos_retarder_angle,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.dark_gos_level0_status,
-            metadata_key=MetadataKey.gos_level0_status,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskUniqueBud(
-            constant_name=BudName.solar_gain_gos_level0_status,
-            metadata_key=MetadataKey.gos_level0_status,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskAverageBud(
-            constant_name=BudName.dark_average_light_level,
-            metadata_key=MetadataKey.light_level,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskAverageBud(
-            constant_name=BudName.solar_gain_average_light_level,
-            metadata_key=MetadataKey.light_level,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskAverageBud(
-            constant_name=BudName.polcal_average_light_level,
-            metadata_key=MetadataKey.light_level,
-            ip_task_types=TaskName.polcal,
-        ),
-        TaskAverageBud(
-            constant_name=BudName.dark_average_telescope_elevation,
-            metadata_key=MetadataKey.elevation,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskAverageBud(
-            constant_name=BudName.solar_gain_average_telescope_elevation,
-            metadata_key=MetadataKey.elevation,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskAverageBud(
-            constant_name=BudName.polcal_average_telescope_elevation,
-            metadata_key=MetadataKey.elevation,
-            ip_task_types=TaskName.polcal,
-        ),
-        TaskAverageBud(
-            constant_name=BudName.dark_average_coude_table_angle,
-            metadata_key=MetadataKey.table_angle,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskAverageBud(
-            constant_name=BudName.solar_gain_average_coude_table_angle,
-            metadata_key=MetadataKey.table_angle,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskAverageBud(
-            constant_name=BudName.polcal_average_coude_table_angle,
-            metadata_key=MetadataKey.table_angle,
-            ip_task_types=TaskName.polcal,
-        ),
-        TaskAverageBud(
-            constant_name=BudName.dark_average_telescope_azimuth,
-            metadata_key=MetadataKey.azimuth,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskAverageBud(
-            constant_name=BudName.solar_gain_average_telescope_azimuth,
-            metadata_key=MetadataKey.azimuth,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskAverageBud(
-            constant_name=BudName.polcal_average_telescope_azimuth,
-            metadata_key=MetadataKey.azimuth,
-            ip_task_types=TaskName.polcal,
-        ),
-        TaskDateBeginBud(
-            constant_name=BudName.dark_date_begin,
-            ip_task_types=TaskName.dark,
-        ),
-        TaskDateBeginBud(
-            constant_name=BudName.solar_gain_date_begin,
-            ip_task_types=TaskName.solar_gain,
-        ),
-        TaskDateBeginBud(
-            constant_name=BudName.polcal_date_begin,
-            ip_task_types=TaskName.polcal,
-        ),
-    ]
-
-
 def default_constant_bud_factory() -> list[S]:
     """Provide default constant buds for use in common parsing tasks."""
-    return dataset_extra_bud_factory() + [
+    return [
         UniqueBud(constant_name=BudName.instrument, metadata_key=MetadataKey.instrument),
         ProposalIdBud(),
         ContributingProposalIdsBud(),

dkist_processing_common/tests/test_fits_access.py

@@ -4,6 +4,7 @@ import numpy as np
 import pytest
 from astropy.io import fits
 
+from dkist_processing_common.models.fits_access import NOT_FOUND_MESSAGE
 from dkist_processing_common.models.fits_access import FitsAccessBase
 from dkist_processing_common.models.fits_access import MetadataKey
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
@@ -98,16 +99,30 @@ def hdu_with_no_data(complete_common_header):
 @pytest.fixture()
 def hdu_with_incomplete_common_header(complete_common_header):
     """
-    An HDU with data and a header missing two of the expected common by-frame keywords
+    An HDU with data and a header missing one of the expected common by-frame keywords
     """
     incomplete_header = complete_common_header
-    incomplete_header.pop(MetadataKey.elevation)
-    incomplete_header.pop(MetadataKey.azimuth)
+    incomplete_header.pop("ELEV_ANG")
+    incomplete_header.pop("TAZIMUTH")
     data = np.arange(9).reshape(3, 3)
     hdu = fits.PrimaryHDU(data, header=incomplete_header)
     return hdu
 
 
+class MetadataKeyWithOptionalKeys(StrEnum):
+    optional1 = "ELEV_ANG"
+    optional2 = "TAZIMUTH"
+
+
+class FitsAccessWithOptionalKeys(FitsAccessBase):
+    def __init__(self, hdu, name):
+        super().__init__(hdu, name)
+        self._set_metadata_key_value(MetadataKeyWithOptionalKeys.optional1, optional=True)
+        self._set_metadata_key_value(
+            MetadataKeyWithOptionalKeys.optional2, optional=True, default="SO_RAD"
+        )
+
+
 @pytest.fixture()
 def fits_file_path(tmp_path, complete_common_header):
     file_path = tmp_path / "foo.fits"
@@ -140,36 +155,35 @@ class MetadataKeyWithNaxisKeys(StrEnum):
 class FitsAccessWithNaxisKeys(FitsAccessBase):
     def __init__(self, hdu, name):
         super().__init__(hdu, name)
-        self.naxis = self.header[MetadataKeyWithNaxisKeys.naxis]
-        self.naxis1 = self.header[MetadataKeyWithNaxisKeys.naxis1]
-        self.naxis2 = self.header[MetadataKeyWithNaxisKeys.naxis2]
+        self._set_metadata_key_value(MetadataKeyWithNaxisKeys.naxis)
+        self._set_metadata_key_value(MetadataKeyWithNaxisKeys.naxis1)
+        self._set_metadata_key_value(MetadataKeyWithNaxisKeys.naxis2)
 
 
 def test_metadata_keys_in_access_bases(hdu_with_no_data):
     """
     Given: a set of metadata key names in the MetadataKey sting enumeration
     When: the FITS access classes define a set of attributes/properties
-    Then: the sets are the same and the values of the attributes/properties are correct
+    Then: the sets are the same
     """
     all_metadata_key_names = {mk.name for mk in MetadataKey}
     hdu = hdu_with_no_data
-    fits_obj = L0FitsAccess(hdu=hdu)
-    fits_access_defined_attributes = {
-        k for k, v in fits_obj.__dict__.items() if k not in ["_hdu", "name", "auto_squeeze"]
-    }
-    l0_access_properties = {k for k, v in L0FitsAccess.__dict__.items() if isinstance(v, property)}
-    l1_access_properties = {k for k, v in L1FitsAccess.__dict__.items() if isinstance(v, property)}
-    fits_access_properties = l0_access_properties | l1_access_properties
-    assert fits_access_defined_attributes | fits_access_properties == all_metadata_key_names
-    for key in fits_access_defined_attributes:
-        assert getattr(fits_obj, key) == fits_obj.header[MetadataKey[key]]
+    l1_properties = {k for k, v in L1FitsAccess.__dict__.items() if isinstance(v, property)}
+    l0_properties = {k for k, v in L0FitsAccess.__dict__.items() if isinstance(v, property)}
+    all_fits_access_properties = l1_properties | l0_properties
+    l1_dict_keys = {k for k, v in L1FitsAccess(hdu=hdu).__dict__.items()}
+    l0_dict_keys = {k for k, v in L0FitsAccess(hdu=hdu).__dict__.items()}
+    all_fits_access_dict_keys = l1_dict_keys | l0_dict_keys
+    instance_dict_keys = {"_hdu", "name", "auto_squeeze"}
+    keys_defined_in_fits_access = all_fits_access_dict_keys | all_fits_access_properties
+    assert all_metadata_key_names == keys_defined_in_fits_access - instance_dict_keys
 
 
 def test_from_single_hdu(hdu_with_complete_common_header):
     """
     Given: an HDU with expected, common by-frame keywords
     When: loading the HDU with the L0FitsAccess class
-    Then: values for common keywords are exposed as properties on the fits_obj class
+    Then: all values for common keywords are exposed as properties on the fits_obj class
     """
     fits_obj = L0FitsAccess(hdu_with_complete_common_header)
     assert fits_obj.elevation == 6.28
@@ -184,7 +198,7 @@ def test_l1_only_fits_access(hdu_with_complete_l1_only_header):
     """
     Given: an HDU with 214 L1-only headers
     When: loading the HDU with the L1FitsAccess class
-    Then: no errors are raised and values are exposed
+    Then: no errors are raised and all values are exposed
     """
     fits_obj = L1FitsAccess(hdu_with_complete_l1_only_header)
     assert fits_obj.elevation == 6.28
@@ -241,6 +255,17 @@ def test_no_header_value(hdu_with_incomplete_common_header):
         _ = L0FitsAccess(hdu_with_incomplete_common_header)
 
 
+def test_default_header_values(hdu_with_incomplete_common_header):
+    """
+    Given: an HDU with a header with missing common by-frame keywords
+    When: processing the HDU with a FITS access class that sets these keywords as optional
+    Then: the correct default values are set
+    """
+    fits_obj = FitsAccessWithOptionalKeys(hdu_with_incomplete_common_header, name="foo")
+    assert fits_obj.optional1 == MetadataKeyWithOptionalKeys.optional1 + NOT_FOUND_MESSAGE
+    assert fits_obj.optional2 == "SO_RAD"
+
+
 def test_as_subclass(hdu_with_complete_common_header):
     """
     Given: an instrument-specific fits_obj class that subclasses L0FitsAccess

dkist_processing_common/tests/test_parse_l0_input_data.py

@@ -97,9 +97,9 @@ class ViSPMetadataKey(StrEnum):
 class ViSPFitsAccess(FitsAccessBase):
     def __init__(self, hdu, name, auto_squeeze=False):
         super().__init__(hdu, name, auto_squeeze=auto_squeeze)
-        self.num_mod: int = self.header[ViSPMetadataKey.num_mod]
-        self.modstate: int = self.header[ViSPMetadataKey.modstate]
-        self.ip_task_type: str = self.header[ViSPMetadataKey.ip_task_type]
+        self._set_metadata_key_value(ViSPMetadataKey.num_mod)
+        self._set_metadata_key_value(ViSPMetadataKey.modstate)
+        self._set_metadata_key_value(ViSPMetadataKey.ip_task_type)
         self.name = name
 
 
@@ -260,52 +260,18 @@ def test_subclass_flowers(visp_parse_inputs_task, max_cs_step_time_sec):
     tag_pot, constant_pot = visp_parse_inputs_task.make_flower_pots()
 
     assert len(tag_pot.stems) == 1
-    assert len(constant_pot.stems) == 64
+    assert len(constant_pot.stems) == 12
     all_flower_names = [StemName.modstate]
     assert sorted([f.stem_name for f in tag_pot.stems]) == sorted(all_flower_names)
     all_bud_names = [b.stem_name for b in default_constant_bud_factory()] + [BudName.num_modstates]
     assert sorted([f.stem_name for f in constant_pot.stems]) == sorted(all_bud_names)
 
 
-def test_dataset_extra_bud_factory(visp_parse_inputs_task, max_cs_step_time_sec):
-    """
-    Given: ParseInputData child class with custom stems
-    When: Making the constant pot
-    Then: The multi-task dataset extra buds are created
-    """
-    _, constant_pot = visp_parse_inputs_task.make_flower_pots()
-    stem_names = [f.stem_name.value for f in constant_pot.stems]
-    bud_name_base = [
-        "DATE_BEGIN",
-        "OBSERVING_PROGRAM_EXECUTION_ID",
-        "NUM_RAW_FRAMES_PER_FPA",
-        "TELESCOPE_TRACKING_MODE",
-        "COUDE_TABLE_TRACKING_MODE",
-        "TELESCOPE_SCANNING_MODE",
-        "AVERAGE_LIGHT_LEVEL",
-        "AVERAGE_TELESCOPE_ELEVATION",
-        "AVERAGE_COUDE_TABLE_ANGLE",
-        "AVERAGE_TELESCOPE_AZIMUTH",
-        "GOS_LEVEL3_STATUS",
-        "GOS_LEVEL3_LAMP_STATUS",
-        "GOS_POLARIZER_STATUS",
-        "GOS_POLARIZER_ANGLE",
-        "GOS_RETARDER_STATUS",
-        "GOS_RETARDER_ANGLE",
-        "GOS_LEVEL0_STATUS",
-    ]
-    for base in bud_name_base:
-        assert "DARK_" + base in stem_names
-        assert "SOLAR_GAIN_" + base in stem_names
-        if "GOS" not in base:
-            assert "POLCAL_" + base in stem_names
-
-
 def test_constants_correct(parse_inputs_task):
     """
     Given: ParseInputData task with a populated constant FlowerPot
     When: Updating pipeline constants
-    Then: A pipeline constant is correctly populated
+    Then: Pipeline constants are correctly populated
     """
     _, constant_pot = parse_inputs_task.make_flower_pots()
     parse_inputs_task.update_constants(constant_pot)

dkist_processing_common/tests/test_quality_mixin.py

@@ -1214,12 +1214,20 @@ def wavecal_weights(wavecal_input_wavelength) -> np.ndarray:
 
 
 @pytest.fixture(scope="session")
-def wavecal_fit_result(wavecal_input_wavelength) -> FitResult:
+def wavecal_fit_result(wavecal_input_wavelength, wavecal_input_spectrum) -> FitResult:
     wavelength_params = WavelengthParameters(
         crpix=1, crval=10.0, dispersion=1, grating_constant=1, order=1, incident_light_angle=0
     )
-    minimizer_result = MinimizerResult(residual=np.random.random(wavecal_input_wavelength.size))
-    return FitResult(wavelength_parameters=wavelength_params, minimizer_result=minimizer_result)
+
+    residuals = np.random.random(wavecal_input_wavelength.size)
+    residuals[-1] = np.nan
+    minimizer_result = MinimizerResult(residual=residuals)
+    return FitResult(
+        wavelength_parameters=wavelength_params,
+        minimizer_result=minimizer_result,
+        input_wavelength_vector=wavecal_input_wavelength,
+        input_spectrum=wavecal_input_spectrum,
+    )
 
 
 @pytest.mark.parametrize(