dkist-processing-common 11.6.0__py3-none-any.whl → 11.7.0rc1__py3-none-any.whl

dkist_processing_common/tasks/parse_l0_input_data.py

@@ -34,16 +34,20 @@ from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.flower_pot import Thorn
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.models.task_name import TaskName
+from dkist_processing_common.parsers.average_bud import TaskAverageBud
 from dkist_processing_common.parsers.experiment_id_bud import ContributingExperimentIdsBud
 from dkist_processing_common.parsers.experiment_id_bud import ExperimentIdBud
+from dkist_processing_common.parsers.id_bud import TaskContributingIdsBud
 from dkist_processing_common.parsers.proposal_id_bud import ContributingProposalIdsBud
 from dkist_processing_common.parsers.proposal_id_bud import ProposalIdBud
 from dkist_processing_common.parsers.time import AverageCadenceBud
 from dkist_processing_common.parsers.time import MaximumCadenceBud
 from dkist_processing_common.parsers.time import MinimumCadenceBud
+from dkist_processing_common.parsers.time import TaskDateBeginBud
 from dkist_processing_common.parsers.time import TaskExposureTimesBud
 from dkist_processing_common.parsers.time import TaskReadoutExpTimesBud
 from dkist_processing_common.parsers.time import VarianceCadenceBud
+from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
 from dkist_processing_common.parsers.unique_bud import UniqueBud
 from dkist_processing_common.tasks.base import WorkflowTaskBase
 
@@ -59,9 +63,288 @@ logger = logging.getLogger(__name__)
 S = TypeVar("S", bound=Stem)
 
 
+def dataset_extra_bud_factory() -> list[S]:
+    """Provide constant buds for use in dataset extras."""
+    return [
+        UniqueBud(constant_name=BudName.camera_id, metadata_key=MetadataKey.camera_id),
+        UniqueBud(constant_name=BudName.camera_name, metadata_key=MetadataKey.camera_name),
+        UniqueBud(
+            constant_name=BudName.camera_bit_depth, metadata_key=MetadataKey.camera_bit_depth
+        ),
+        UniqueBud(
+            constant_name=BudName.hardware_binning_x, metadata_key=MetadataKey.hardware_binning_x
+        ),
+        UniqueBud(
+            constant_name=BudName.hardware_binning_y, metadata_key=MetadataKey.hardware_binning_x
+        ),
+        UniqueBud(
+            constant_name=BudName.software_binning_x, metadata_key=MetadataKey.software_binning_x
+        ),
+        UniqueBud(
+            constant_name=BudName.software_binning_y, metadata_key=MetadataKey.software_binning_y
+        ),
+        UniqueBud(
+            constant_name=BudName.hls_version,
+            metadata_key=MetadataKey.hls_version,
+        ),
+        TaskContributingIdsBud(
+            constant_name=BudName.dark_observing_program_execution_id,
+            metadata_key=MetadataKey.observing_program_execution_id,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskContributingIdsBud(
+            constant_name=BudName.solar_gain_observing_program_execution_id,
+            metadata_key=MetadataKey.observing_program_execution_id,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskContributingIdsBud(
+            constant_name=BudName.polcal_observing_program_execution_id,
+            metadata_key=MetadataKey.observing_program_execution_id,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_num_raw_frames_per_fpa,
+            metadata_key=MetadataKey.num_raw_frames_per_fpa,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_num_raw_frames_per_fpa,
+            metadata_key=MetadataKey.num_raw_frames_per_fpa,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_num_raw_frames_per_fpa,
+            metadata_key=MetadataKey.num_raw_frames_per_fpa,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_telescope_tracking_mode,
+            metadata_key=MetadataKey.telescope_tracking_mode,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_telescope_tracking_mode,
+            metadata_key=MetadataKey.telescope_tracking_mode,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_telescope_tracking_mode,
+            metadata_key=MetadataKey.telescope_tracking_mode,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_coude_table_tracking_mode,
+            metadata_key=MetadataKey.coude_table_tracking_mode,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_coude_table_tracking_mode,
+            metadata_key=MetadataKey.coude_table_tracking_mode,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_coude_table_tracking_mode,
+            metadata_key=MetadataKey.coude_table_tracking_mode,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_telescope_scanning_mode,
+            metadata_key=MetadataKey.telescope_scanning_mode,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_telescope_scanning_mode,
+            metadata_key=MetadataKey.telescope_scanning_mode,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_telescope_scanning_mode,
+            metadata_key=MetadataKey.telescope_scanning_mode,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_level3_status,
+            metadata_key=MetadataKey.gos_level3_status,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_level3_status,
+            metadata_key=MetadataKey.gos_level3_status,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_gos_level3_status,
+            metadata_key=MetadataKey.gos_level3_status,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_level3_lamp_status,
+            metadata_key=MetadataKey.gos_level3_lamp_status,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_level3_lamp_status,
+            metadata_key=MetadataKey.gos_level3_lamp_status,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_gos_level3_lamp_status,
+            metadata_key=MetadataKey.gos_level3_lamp_status,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_polarizer_status,
+            metadata_key=MetadataKey.gos_polarizer_status,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_polarizer_status,
+            metadata_key=MetadataKey.gos_polarizer_status,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_gos_polarizer_status,
+            metadata_key=MetadataKey.gos_polarizer_status,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_polarizer_angle,
+            metadata_key=MetadataKey.gos_polarizer_angle,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_polarizer_angle,
+            metadata_key=MetadataKey.gos_polarizer_angle,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_gos_polarizer_angle,
+            metadata_key=MetadataKey.gos_polarizer_angle,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_retarder_status,
+            metadata_key=MetadataKey.gos_retarder_status,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_retarder_status,
+            metadata_key=MetadataKey.gos_retarder_status,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_gos_retarder_status,
+            metadata_key=MetadataKey.gos_retarder_status,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_retarder_angle,
+            metadata_key=MetadataKey.gos_retarder_angle,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_retarder_angle,
+            metadata_key=MetadataKey.gos_retarder_angle,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_gos_retarder_angle,
+            metadata_key=MetadataKey.gos_retarder_angle,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.dark_gos_level0_status,
+            metadata_key=MetadataKey.gos_level0_status,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.solar_gain_gos_level0_status,
+            metadata_key=MetadataKey.gos_level0_status,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskUniqueBud(
+            constant_name=BudName.polcal_gos_level0_status,
+            metadata_key=MetadataKey.gos_level0_status,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.dark_average_light_level,
+            metadata_key=MetadataKey.light_level,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.solar_gain_average_light_level,
+            metadata_key=MetadataKey.light_level,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.polcal_average_light_level,
+            metadata_key=MetadataKey.light_level,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.dark_average_telescope_elevation,
+            metadata_key=MetadataKey.elevation,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.solar_gain_average_telescope_elevation,
+            metadata_key=MetadataKey.elevation,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.polcal_average_telescope_elevation,
+            metadata_key=MetadataKey.elevation,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.dark_average_coude_table_angle,
+            metadata_key=MetadataKey.table_angle,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.solar_gain_average_coude_table_angle,
+            metadata_key=MetadataKey.table_angle,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.polcal_average_coude_table_angle,
+            metadata_key=MetadataKey.table_angle,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.dark_average_telescope_azimuth,
+            metadata_key=MetadataKey.azimuth,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.solar_gain_average_telescope_azimuth,
+            metadata_key=MetadataKey.azimuth,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskAverageBud(
+            constant_name=BudName.polcal_average_telescope_azimuth,
+            metadata_key=MetadataKey.azimuth,
+            ip_task_types=TaskName.polcal,
+        ),
+        TaskDateBeginBud(
+            constant_name=BudName.dark_date_begin,
+            ip_task_types=TaskName.dark,
+        ),
+        TaskDateBeginBud(
+            constant_name=BudName.solar_gain_date_begin,
+            ip_task_types=TaskName.solar_gain,
+        ),
+        TaskDateBeginBud(
+            constant_name=BudName.polcal_date_begin,
+            ip_task_types=TaskName.polcal,
+        ),
+    ]
+
+
 def default_constant_bud_factory() -> list[S]:
     """Provide default constant buds for use in common parsing tasks."""
-    return [
+    return dataset_extra_bud_factory() + [
         UniqueBud(constant_name=BudName.instrument, metadata_key=MetadataKey.instrument),
         ProposalIdBud(),
         ContributingProposalIdsBud(),

dkist_processing_common/tests/test_fits_access.py

@@ -4,7 +4,6 @@ import numpy as np
 import pytest
 from astropy.io import fits
 
-from dkist_processing_common.models.fits_access import NOT_FOUND_MESSAGE
 from dkist_processing_common.models.fits_access import FitsAccessBase
 from dkist_processing_common.models.fits_access import MetadataKey
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
@@ -99,30 +98,16 @@ def hdu_with_no_data(complete_common_header):
 @pytest.fixture()
 def hdu_with_incomplete_common_header(complete_common_header):
     """
-    An HDU with data and a header missing one of the expected common by-frame keywords
+    An HDU with data and a header missing two of the expected common by-frame keywords
     """
     incomplete_header = complete_common_header
-    incomplete_header.pop("ELEV_ANG")
-    incomplete_header.pop("TAZIMUTH")
+    incomplete_header.pop(MetadataKey.elevation)
+    incomplete_header.pop(MetadataKey.azimuth)
     data = np.arange(9).reshape(3, 3)
     hdu = fits.PrimaryHDU(data, header=incomplete_header)
     return hdu
 
 
-class MetadataKeyWithOptionalKeys(StrEnum):
-    optional1 = "ELEV_ANG"
-    optional2 = "TAZIMUTH"
-
-
-class FitsAccessWithOptionalKeys(FitsAccessBase):
-    def __init__(self, hdu, name):
-        super().__init__(hdu, name)
-        self._set_metadata_key_value(MetadataKeyWithOptionalKeys.optional1, optional=True)
-        self._set_metadata_key_value(
-            MetadataKeyWithOptionalKeys.optional2, optional=True, default="SO_RAD"
-        )
-
-
 @pytest.fixture()
 def fits_file_path(tmp_path, complete_common_header):
     file_path = tmp_path / "foo.fits"
@@ -155,35 +140,36 @@ class MetadataKeyWithNaxisKeys(StrEnum):
 class FitsAccessWithNaxisKeys(FitsAccessBase):
     def __init__(self, hdu, name):
         super().__init__(hdu, name)
-        self._set_metadata_key_value(MetadataKeyWithNaxisKeys.naxis)
-        self._set_metadata_key_value(MetadataKeyWithNaxisKeys.naxis1)
-        self._set_metadata_key_value(MetadataKeyWithNaxisKeys.naxis2)
+        self.naxis = self.header[MetadataKeyWithNaxisKeys.naxis]
+        self.naxis1 = self.header[MetadataKeyWithNaxisKeys.naxis1]
+        self.naxis2 = self.header[MetadataKeyWithNaxisKeys.naxis2]
 
 
 def test_metadata_keys_in_access_bases(hdu_with_no_data):
     """
     Given: a set of metadata key names in the MetadataKey sting enumeration
     When: the FITS access classes define a set of attributes/properties
-    Then: the sets are the same
+    Then: the sets are the same and the values of the attributes/properties are correct
     """
     all_metadata_key_names = {mk.name for mk in MetadataKey}
     hdu = hdu_with_no_data
-    l1_properties = {k for k, v in L1FitsAccess.__dict__.items() if isinstance(v, property)}
-    l0_properties = {k for k, v in L0FitsAccess.__dict__.items() if isinstance(v, property)}
-    all_fits_access_properties = l1_properties | l0_properties
-    l1_dict_keys = {k for k, v in L1FitsAccess(hdu=hdu).__dict__.items()}
-    l0_dict_keys = {k for k, v in L0FitsAccess(hdu=hdu).__dict__.items()}
-    all_fits_access_dict_keys = l1_dict_keys | l0_dict_keys
-    instance_dict_keys = {"_hdu", "name", "auto_squeeze"}
-    keys_defined_in_fits_access = all_fits_access_dict_keys | all_fits_access_properties
-    assert all_metadata_key_names == keys_defined_in_fits_access - instance_dict_keys
+    fits_obj = L0FitsAccess(hdu=hdu)
+    fits_access_defined_attributes = {
+        k for k, v in fits_obj.__dict__.items() if k not in ["_hdu", "name", "auto_squeeze"]
+    }
+    l0_access_properties = {k for k, v in L0FitsAccess.__dict__.items() if isinstance(v, property)}
+    l1_access_properties = {k for k, v in L1FitsAccess.__dict__.items() if isinstance(v, property)}
+    fits_access_properties = l0_access_properties | l1_access_properties
+    assert fits_access_defined_attributes | fits_access_properties == all_metadata_key_names
+    for key in fits_access_defined_attributes:
+        assert getattr(fits_obj, key) == fits_obj.header[MetadataKey[key]]
 
 
 def test_from_single_hdu(hdu_with_complete_common_header):
     """
     Given: an HDU with expected, common by-frame keywords
     When: loading the HDU with the L0FitsAccess class
-    Then: all values for common keywords are exposed as properties on the fits_obj class
+    Then: values for common keywords are exposed as properties on the fits_obj class
     """
     fits_obj = L0FitsAccess(hdu_with_complete_common_header)
     assert fits_obj.elevation == 6.28
@@ -198,7 +184,7 @@ def test_l1_only_fits_access(hdu_with_complete_l1_only_header):
     """
     Given: an HDU with 214 L1-only headers
     When: loading the HDU with the L1FitsAccess class
-    Then: no errors are raised and all values are exposed
+    Then: no errors are raised and values are exposed
     """
     fits_obj = L1FitsAccess(hdu_with_complete_l1_only_header)
     assert fits_obj.elevation == 6.28
@@ -255,17 +241,6 @@ def test_no_header_value(hdu_with_incomplete_common_header):
         _ = L0FitsAccess(hdu_with_incomplete_common_header)
 
 
-def test_default_header_values(hdu_with_incomplete_common_header):
-    """
-    Given: an HDU with a header with missing common by-frame keywords
-    When: processing the HDU with a FITS access class that sets these keywords as optional
-    Then: the correct default values are set
-    """
-    fits_obj = FitsAccessWithOptionalKeys(hdu_with_incomplete_common_header, name="foo")
-    assert fits_obj.optional1 == MetadataKeyWithOptionalKeys.optional1 + NOT_FOUND_MESSAGE
-    assert fits_obj.optional2 == "SO_RAD"
-
-
 def test_as_subclass(hdu_with_complete_common_header):
     """
     Given: an instrument-specific fits_obj class that subclasses L0FitsAccess

dkist_processing_common/tests/test_parse_l0_input_data.py

@@ -97,9 +97,9 @@ class ViSPMetadataKey(StrEnum):
 class ViSPFitsAccess(FitsAccessBase):
     def __init__(self, hdu, name, auto_squeeze=False):
         super().__init__(hdu, name, auto_squeeze=auto_squeeze)
-        self._set_metadata_key_value(ViSPMetadataKey.num_mod)
-        self._set_metadata_key_value(ViSPMetadataKey.modstate)
-        self._set_metadata_key_value(ViSPMetadataKey.ip_task_type)
+        self.num_mod: int = self.header[ViSPMetadataKey.num_mod]
+        self.modstate: int = self.header[ViSPMetadataKey.modstate]
+        self.ip_task_type: str = self.header[ViSPMetadataKey.ip_task_type]
         self.name = name
 
 
@@ -260,18 +260,52 @@ def test_subclass_flowers(visp_parse_inputs_task, max_cs_step_time_sec):
     tag_pot, constant_pot = visp_parse_inputs_task.make_flower_pots()
 
     assert len(tag_pot.stems) == 1
-    assert len(constant_pot.stems) == 12
+    assert len(constant_pot.stems) == 71
     all_flower_names = [StemName.modstate]
     assert sorted([f.stem_name for f in tag_pot.stems]) == sorted(all_flower_names)
     all_bud_names = [b.stem_name for b in default_constant_bud_factory()] + [BudName.num_modstates]
     assert sorted([f.stem_name for f in constant_pot.stems]) == sorted(all_bud_names)
 
 
+def test_dataset_extra_bud_factory(visp_parse_inputs_task, max_cs_step_time_sec):
+    """
+    Given: ParseInputData child class with custom stems
+    When: Making the constant pot
+    Then: The multi-task dataset extra buds are all created
+    """
+    _, constant_pot = visp_parse_inputs_task.make_flower_pots()
+    stem_names = [f.stem_name.value for f in constant_pot.stems]
+    bud_name_base = [
+        "DATE_BEGIN",
+        "OBSERVING_PROGRAM_EXECUTION_ID",
+        "NUM_RAW_FRAMES_PER_FPA",
+        "TELESCOPE_TRACKING_MODE",
+        "COUDE_TABLE_TRACKING_MODE",
+        "TELESCOPE_SCANNING_MODE",
+        "AVERAGE_LIGHT_LEVEL",
+        "AVERAGE_TELESCOPE_ELEVATION",
+        "AVERAGE_COUDE_TABLE_ANGLE",
+        "AVERAGE_TELESCOPE_AZIMUTH",
+        "GOS_LEVEL3_STATUS",
+        "GOS_LEVEL3_LAMP_STATUS",
+        "GOS_POLARIZER_STATUS",
+        "GOS_POLARIZER_ANGLE",
+        "GOS_RETARDER_STATUS",
+        "GOS_RETARDER_ANGLE",
+        "GOS_LEVEL0_STATUS",
+    ]
+    bud_name_prefix = ["DARK_", "SOLAR_GAIN_", "POLCAL_"]
+    for prefix in bud_name_prefix:
+        for base in bud_name_base:
+            print(prefix + base)
+            assert prefix + base in stem_names
+
+
 def test_constants_correct(parse_inputs_task):
     """
     Given: ParseInputData task with a populated constant FlowerPot
     When: Updating pipeline constants
-    Then: Pipeline constants are correctly populated
+    Then: A pipeline constant is correctly populated
     """
     _, constant_pot = parse_inputs_task.make_flower_pots()
     parse_inputs_task.update_constants(constant_pot)