dkist-processing-common 11.7.0rc2__tar.gz → 11.7.0rc4__tar.gz

{dkist_processing_common-11.7.0rc2 → dkist_processing_common-11.7.0rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 11.7.0rc2
+Version: 11.7.0rc4
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-11.7.0rc2 → dkist_processing_common-11.7.0rc4}/dkist_processing_common/models/constants.py

@@ -44,9 +44,9 @@ class BudName(StrEnum):
     software_binning_y = "SOFTWARE_BINNING_Y"
     hls_version = "HLS_VERSION"
     # Multi-task buds start here:
-    dark_observing_program_execution_id = "DARK_OBSERVING_PROGRAM_EXECUTION_ID"
-    solar_gain_observing_program_execution_id = "SOLAR_GAIN_OBSERVING_PROGRAM_EXECUTION_ID"
-    polcal_observing_program_execution_id = "POLCAL_OBSERVING_PROGRAM_EXECUTION_ID"
+    dark_observing_program_execution_ids = "DARK_OBSERVING_PROGRAM_EXECUTION_IDS"
+    solar_gain_observing_program_execution_ids = "SOLAR_GAIN_OBSERVING_PROGRAM_EXECUTION_IDS"
+    polcal_observing_program_execution_ids = "POLCAL_OBSERVING_PROGRAM_EXECUTION_IDS"
     dark_date_begin = "DARK_DATE_BEGIN"
     solar_gain_date_begin = "SOLAR_GAIN_DATE_BEGIN"
     polcal_date_begin = "POLCAL_DATE_BEGIN"
@@ -285,21 +285,21 @@ class ConstantsBase:
     # Multi-task constants start here:
 
     @property
-    def dark_observing_program_execution_id(self) -> list[str]:
-        """Return the observing program execution id constant for the dark task."""
-        observing_programs = self._db_dict[BudName.dark_observing_program_execution_id]
+    def dark_observing_program_execution_ids(self) -> list[str]:
+        """Return the observing program execution ids constant for the dark task."""
+        observing_programs = self._db_dict[BudName.dark_observing_program_execution_ids]
         return list(observing_programs)
 
     @property
-    def solar_gain_observing_program_execution_id(self) -> list[str]:
-        """Return the observing program execution id constant for the solar_gain task."""
-        observing_programs = self._db_dict[BudName.solar_gain_observing_program_execution_id]
+    def solar_gain_observing_program_execution_ids(self) -> list[str]:
+        """Return the observing program execution ids constant for the solar_gain task."""
+        observing_programs = self._db_dict[BudName.solar_gain_observing_program_execution_ids]
         return list(observing_programs)
 
     @property
-    def polcal_observing_program_execution_id(self) -> list[str]:
-        """Return the observing program execution id constant."""
-        observing_programs = self._db_dict[BudName.polcal_observing_program_execution_id]
+    def polcal_observing_program_execution_ids(self) -> list[str]:
+        """Return the observing program execution ids constant."""
+        observing_programs = self._db_dict[BudName.polcal_observing_program_execution_ids]
         return list(observing_programs)
 
     @property

dkist_processing_common-11.7.0rc4/dkist_processing_common/parsers/observing_program_id_bud.py

@@ -0,0 +1,24 @@
+"""Observing Program Id parser."""
+
+from typing import Callable
+
+from dkist_processing_common.models.fits_access import MetadataKey
+from dkist_processing_common.parsers.id_bud import TaskContributingIdsBud
+from dkist_processing_common.parsers.task import passthrough_header_ip_task
+
+
+class TaskContributingObservingProgramExecutionIdsBud(TaskContributingIdsBud):
+    """Class to create a Bud for the supporting observing_program_ids."""
+
+    def __init__(
+        self,
+        constant_name: str,
+        ip_task_types: str | list[str],
+        task_type_parsing_function: Callable = passthrough_header_ip_task,
+    ):
+        super().__init__(
+            constant_name=constant_name,
+            metadata_key=MetadataKey.observing_program_execution_id,
+            ip_task_types=ip_task_types,
+            task_type_parsing_function=task_type_parsing_function,
+        )

{dkist_processing_common-11.7.0rc2 → dkist_processing_common-11.7.0rc4}/dkist_processing_common/parsers/time.py

@@ -66,7 +66,7 @@ class TaskDatetimeBudBase(Stem):
         stem_name: str,
         metadata_key: str | StrEnum,
         ip_task_types: str | list[str],
-        header_task_parsing_func: Callable = passthrough_header_ip_task,
+        task_type_parsing_function: Callable = passthrough_header_ip_task,
     ):
         super().__init__(stem_name=stem_name)
 
@@ -76,7 +76,7 @@ class TaskDatetimeBudBase(Stem):
         if isinstance(ip_task_types, str):
             ip_task_types = [ip_task_types]
         self.ip_task_types = [task.casefold() for task in ip_task_types]
-        self.header_parsing_function = header_task_parsing_func
+        self.header_parsing_function = task_type_parsing_function
 
     def setter(self, fits_obj: L0FitsAccess) -> float | Type[SpilledDirt]:
         """
@@ -218,11 +218,17 @@ class VarianceCadenceBud(CadenceBudBase):
 class TaskDateBeginBud(TaskDatetimeBudBase):
     """Class for the date begin task Bud."""
 
-    def __init__(self, constant_name: str, ip_task_types: str | list[str]):
+    def __init__(
+        self,
+        constant_name: str,
+        ip_task_types: str | list[str],
+        task_type_parsing_function: Callable = passthrough_header_ip_task,
+    ):
         super().__init__(
             stem_name=constant_name,
             metadata_key=MetadataKey.time_obs,
             ip_task_types=ip_task_types,
+            task_type_parsing_function=task_type_parsing_function,
         )
 
     def getter(self, key) -> str:

{dkist_processing_common-11.7.0rc2 → dkist_processing_common-11.7.0rc4}/dkist_processing_common/tasks/parse_l0_input_data.py

@@ -37,9 +37,12 @@ from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_common.parsers.average_bud import TaskAverageBud
 from dkist_processing_common.parsers.experiment_id_bud import ContributingExperimentIdsBud
 from dkist_processing_common.parsers.experiment_id_bud import ExperimentIdBud
-from dkist_processing_common.parsers.id_bud import TaskContributingIdsBud
+from dkist_processing_common.parsers.observing_program_id_bud import (
+    TaskContributingObservingProgramExecutionIdsBud,
+)
 from dkist_processing_common.parsers.proposal_id_bud import ContributingProposalIdsBud
 from dkist_processing_common.parsers.proposal_id_bud import ProposalIdBud
+from dkist_processing_common.parsers.task import parse_header_ip_task_with_gains
 from dkist_processing_common.parsers.time import AverageCadenceBud
 from dkist_processing_common.parsers.time import MaximumCadenceBud
 from dkist_processing_common.parsers.time import MinimumCadenceBud
@@ -87,19 +90,17 @@ def dataset_extra_bud_factory() -> list[S]:
             constant_name=BudName.hls_version,
             metadata_key=MetadataKey.hls_version,
         ),
-        TaskContributingIdsBud(
-            constant_name=BudName.dark_observing_program_execution_id,
-            metadata_key=MetadataKey.observing_program_execution_id,
+        TaskContributingObservingProgramExecutionIdsBud(
+            constant_name=BudName.dark_observing_program_execution_ids,
             ip_task_types=TaskName.dark,
         ),
-        TaskContributingIdsBud(
-            constant_name=BudName.solar_gain_observing_program_execution_id,
-            metadata_key=MetadataKey.observing_program_execution_id,
+        TaskContributingObservingProgramExecutionIdsBud(
+            constant_name=BudName.solar_gain_observing_program_execution_ids,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
-        TaskContributingIdsBud(
-            constant_name=BudName.polcal_observing_program_execution_id,
-            metadata_key=MetadataKey.observing_program_execution_id,
+        TaskContributingObservingProgramExecutionIdsBud(
+            constant_name=BudName.polcal_observing_program_execution_ids,
             ip_task_types=TaskName.polcal,
         ),
         TaskUniqueBud(
@@ -111,51 +112,40 @@ def dataset_extra_bud_factory() -> list[S]:
             constant_name=BudName.solar_gain_num_raw_frames_per_fpa,
             metadata_key=MetadataKey.num_raw_frames_per_fpa,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskUniqueBud(
             constant_name=BudName.polcal_num_raw_frames_per_fpa,
             metadata_key=MetadataKey.num_raw_frames_per_fpa,
             ip_task_types=TaskName.polcal,
         ),
-        TaskUniqueBud(
-            constant_name=BudName.dark_telescope_tracking_mode,
-            metadata_key=MetadataKey.telescope_tracking_mode,
-            ip_task_types=TaskName.dark,
-        ),
         TaskUniqueBud(
             constant_name=BudName.solar_gain_telescope_tracking_mode,
             metadata_key=MetadataKey.telescope_tracking_mode,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskUniqueBud(
             constant_name=BudName.polcal_telescope_tracking_mode,
             metadata_key=MetadataKey.telescope_tracking_mode,
             ip_task_types=TaskName.polcal,
         ),
-        TaskUniqueBud(
-            constant_name=BudName.dark_coude_table_tracking_mode,
-            metadata_key=MetadataKey.coude_table_tracking_mode,
-            ip_task_types=TaskName.dark,
-        ),
         TaskUniqueBud(
             constant_name=BudName.solar_gain_coude_table_tracking_mode,
             metadata_key=MetadataKey.coude_table_tracking_mode,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskUniqueBud(
             constant_name=BudName.polcal_coude_table_tracking_mode,
             metadata_key=MetadataKey.coude_table_tracking_mode,
             ip_task_types=TaskName.polcal,
         ),
-        TaskUniqueBud(
-            constant_name=BudName.dark_telescope_scanning_mode,
-            metadata_key=MetadataKey.telescope_scanning_mode,
-            ip_task_types=TaskName.dark,
-        ),
         TaskUniqueBud(
             constant_name=BudName.solar_gain_telescope_scanning_mode,
             metadata_key=MetadataKey.telescope_scanning_mode,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskUniqueBud(
             constant_name=BudName.polcal_telescope_scanning_mode,
@@ -171,6 +161,7 @@ def dataset_extra_bud_factory() -> list[S]:
             constant_name=BudName.solar_gain_gos_level3_status,
             metadata_key=MetadataKey.gos_level3_status,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskUniqueBud(
             constant_name=BudName.dark_gos_level3_lamp_status,
@@ -181,6 +172,7 @@ def dataset_extra_bud_factory() -> list[S]:
             constant_name=BudName.solar_gain_gos_level3_lamp_status,
             metadata_key=MetadataKey.gos_level3_lamp_status,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskUniqueBud(
             constant_name=BudName.dark_gos_polarizer_status,
@@ -191,6 +183,7 @@ def dataset_extra_bud_factory() -> list[S]:
             constant_name=BudName.solar_gain_gos_polarizer_status,
             metadata_key=MetadataKey.gos_polarizer_status,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskUniqueBud(
             constant_name=BudName.dark_gos_polarizer_angle,
@@ -201,6 +194,7 @@ def dataset_extra_bud_factory() -> list[S]:
             constant_name=BudName.solar_gain_gos_polarizer_angle,
             metadata_key=MetadataKey.gos_polarizer_angle,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskUniqueBud(
             constant_name=BudName.dark_gos_retarder_status,
@@ -211,6 +205,7 @@ def dataset_extra_bud_factory() -> list[S]:
             constant_name=BudName.solar_gain_gos_retarder_status,
             metadata_key=MetadataKey.gos_retarder_status,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskUniqueBud(
             constant_name=BudName.dark_gos_retarder_angle,
@@ -221,6 +216,7 @@ def dataset_extra_bud_factory() -> list[S]:
             constant_name=BudName.solar_gain_gos_retarder_angle,
             metadata_key=MetadataKey.gos_retarder_angle,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskUniqueBud(
             constant_name=BudName.dark_gos_level0_status,
@@ -231,6 +227,7 @@ def dataset_extra_bud_factory() -> list[S]:
             constant_name=BudName.solar_gain_gos_level0_status,
             metadata_key=MetadataKey.gos_level0_status,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskAverageBud(
             constant_name=BudName.dark_average_light_level,
@@ -241,6 +238,7 @@ def dataset_extra_bud_factory() -> list[S]:
             constant_name=BudName.solar_gain_average_light_level,
             metadata_key=MetadataKey.light_level,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskAverageBud(
             constant_name=BudName.polcal_average_light_level,
@@ -256,6 +254,7 @@ def dataset_extra_bud_factory() -> list[S]:
             constant_name=BudName.solar_gain_average_telescope_elevation,
             metadata_key=MetadataKey.elevation,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskAverageBud(
             constant_name=BudName.polcal_average_telescope_elevation,
@@ -271,6 +270,7 @@ def dataset_extra_bud_factory() -> list[S]:
             constant_name=BudName.solar_gain_average_coude_table_angle,
             metadata_key=MetadataKey.table_angle,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskAverageBud(
             constant_name=BudName.polcal_average_coude_table_angle,
@@ -286,6 +286,7 @@ def dataset_extra_bud_factory() -> list[S]:
             constant_name=BudName.solar_gain_average_telescope_azimuth,
             metadata_key=MetadataKey.azimuth,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskAverageBud(
             constant_name=BudName.polcal_average_telescope_azimuth,
@@ -299,6 +300,7 @@ def dataset_extra_bud_factory() -> list[S]:
         TaskDateBeginBud(
             constant_name=BudName.solar_gain_date_begin,
             ip_task_types=TaskName.solar_gain,
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskDateBeginBud(
             constant_name=BudName.polcal_date_begin,

{dkist_processing_common-11.7.0rc2 → dkist_processing_common-11.7.0rc4}/dkist_processing_common/tests/test_parse_l0_input_data.py

@@ -260,7 +260,7 @@ def test_subclass_flowers(visp_parse_inputs_task, max_cs_step_time_sec):
     tag_pot, constant_pot = visp_parse_inputs_task.make_flower_pots()
 
     assert len(tag_pot.stems) == 1
-    assert len(constant_pot.stems) == 64
+    assert len(constant_pot.stems) == 61
     all_flower_names = [StemName.modstate]
     assert sorted([f.stem_name for f in tag_pot.stems]) == sorted(all_flower_names)
     all_bud_names = [b.stem_name for b in default_constant_bud_factory()] + [BudName.num_modstates]
@@ -277,7 +277,7 @@ def test_dataset_extra_bud_factory(visp_parse_inputs_task, max_cs_step_time_sec)
     stem_names = [f.stem_name.value for f in constant_pot.stems]
     bud_name_base = [
         "DATE_BEGIN",
-        "OBSERVING_PROGRAM_EXECUTION_ID",
+        "OBSERVING_PROGRAM_EXECUTION_IDS",
         "NUM_RAW_FRAMES_PER_FPA",
         "TELESCOPE_TRACKING_MODE",
         "COUDE_TABLE_TRACKING_MODE",
@@ -295,8 +295,9 @@ def test_dataset_extra_bud_factory(visp_parse_inputs_task, max_cs_step_time_sec)
         "GOS_LEVEL0_STATUS",
     ]
     for base in bud_name_base:
-        assert "DARK_" + base in stem_names
         assert "SOLAR_GAIN_" + base in stem_names
+        if "MODE" not in base:
+            assert "DARK_" + base in stem_names
         if "GOS" not in base:
             assert "POLCAL_" + base in stem_names
 

{dkist_processing_common-11.7.0rc2 → dkist_processing_common-11.7.0rc4}/dkist_processing_common/tests/test_stems.py

@@ -19,6 +19,9 @@ from dkist_processing_common.parsers.experiment_id_bud import ExperimentIdBud
 from dkist_processing_common.parsers.id_bud import TaskContributingIdsBud
 from dkist_processing_common.parsers.near_bud import NearFloatBud
 from dkist_processing_common.parsers.near_bud import TaskNearFloatBud
+from dkist_processing_common.parsers.observing_program_id_bud import (
+    TaskContributingObservingProgramExecutionIdsBud,
+)
 from dkist_processing_common.parsers.proposal_id_bud import ContributingProposalIdsBud
 from dkist_processing_common.parsers.proposal_id_bud import ProposalIdBud
 from dkist_processing_common.parsers.retarder import RetarderNameBud
@@ -51,6 +54,7 @@ class FitsReaderMetadataKey(StrEnum):
     near_thing = "near"
     proposal_id = "ID___013"
     experiment_id = "ID___012"
+    observing_program_execution_id = "ID___008"
     ip_task_type = "DKIST004"
     ip_start_time = "DKIST011"
     fpa_exposure_time_ms = "XPOSURE"
@@ -77,6 +81,9 @@ class FitsReader(FitsAccessBase):
         self.name = name
         self.proposal_id: str = self.header.get(FitsReaderMetadataKey.proposal_id)
         self.experiment_id: str = self.header.get(FitsReaderMetadataKey.experiment_id)
+        self.observing_program_execution_id: str = self.header.get(
+            FitsReaderMetadataKey.observing_program_execution_id
+        )
         self.ip_task_type: str = self.header.get(FitsReaderMetadataKey.ip_task_type)
         self.ip_start_time: str = self.header.get(FitsReaderMetadataKey.ip_start_time)
         self.fpa_exposure_time_ms: float = self.header.get(
@@ -113,6 +120,7 @@ def basic_header_objs():
                 "DKIST004": "observe",
                 "ID___012": "experiment_id_1",
                 "ID___013": "proposal_id_1",
+                "ID___008": "observing_program_execution_id_1",
                 "XPOSURE": 0.0013000123,
                 "TEXPOSUR": 10.0,
                 "NSUMEXP": 3,
@@ -131,6 +139,7 @@ def basic_header_objs():
                 "DKIST004": "observe",
                 "ID___012": "experiment_id_1",
                 "ID___013": "proposal_id_1",
+                "ID___008": "observing_program_execution_id_2",
                 "XPOSURE": 0.0013000987,
                 "TEXPOSUR": 10.0,
                 "NSUMEXP": 3,
@@ -150,6 +159,7 @@ def basic_header_objs():
                 "DKIST004": "dark",
                 "ID___012": "experiment_id_2",
                 "ID___013": "proposal_id_2",
+                "ID___008": "observing_program_execution_id_2",
                 "XPOSURE": 12.345,
                 "TEXPOSUR": 1.123456789,
                 "NSUMEXP": 1,
@@ -170,6 +180,7 @@ def basic_header_objs():
                 "DKIST004": "observe",
                 "ID___012": "experiment_id_1",
                 "ID___013": "proposal_id_1",
+                "ID___008": "observing_program_execution_id_1",
                 "XPOSURE": 100.0,
                 "TEXPOSUR": 11.0,
                 "NSUMEXP": 4,
@@ -185,6 +196,7 @@ def basic_header_objs():
             {
                 "DKIST004": "gain",
                 "ID___013": "proposal_id_1",
+                "ID___008": "observing_program_execution_id_1",
                 "id_key": 0,
                 "constant": 6.28,
                 "near": 1.23,
@@ -603,6 +615,29 @@ def test_task_contributing_ids_bud(basic_header_objs):
     assert sorted(list(petal[0].value)) == ["experiment_id_2"]
 
 
+def test_task_contributing_observing_program_ids_bud(basic_header_objs):
+    """
+    Given: A set of headers with observing program ID values for different tasks
+    When: Ingesting the headers with a TaskContributingIdsBud for a task type
+    Then: The Bud's petal is just the experiment ID for the that task type
+    """
+    bud = TaskContributingObservingProgramExecutionIdsBud(
+        constant_name="NOT_A_REAL_BUD",
+        ip_task_types=TaskName.observe,
+    )
+    assert bud.stem_name == "NOT_A_REAL_BUD"
+    for fo in basic_header_objs:
+        key = fo.name
+        bud.update(key, fo)
+
+    petal = list(bud.petals)
+    assert len(petal) == 1
+    assert sorted(list(petal[0].value)) == [
+        "observing_program_execution_id_1",
+        "observing_program_execution_id_2",
+    ]
+
+
 def test_exp_time_flower(basic_header_objs):
     """
     Given: A set of filepaths and associated headers with XPOSURE keywords

{dkist_processing_common-11.7.0rc2 → dkist_processing_common-11.7.0rc4}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 11.7.0rc2
+Version: 11.7.0rc4
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-11.7.0rc2 → dkist_processing_common-11.7.0rc4}/dkist_processing_common.egg-info/SOURCES.txt

@@ -65,6 +65,7 @@ dkist_processing_common/parsers/id_bud.py
 dkist_processing_common/parsers/l0_fits_access.py
 dkist_processing_common/parsers/l1_fits_access.py
 dkist_processing_common/parsers/near_bud.py
+dkist_processing_common/parsers/observing_program_id_bud.py
 dkist_processing_common/parsers/proposal_id_bud.py
 dkist_processing_common/parsers/quality.py
 dkist_processing_common/parsers/retarder.py