dkist-processing-common 14.1.1__tar.gz → 14.2.0__tar.gz

{dkist_processing_common-14.1.1 → dkist_processing_common-14.2.0}/CHANGELOG.rst

@@ -1,3 +1,24 @@
+v14.2.0 (2026-05-12)
+====================
+
+Features
+--------
+
+- Add the `TaskMostFrequentBud`. This parser is like the `TaskUniqueBud` except that when there is more than one unique
+  value the most frequent value is returned. (`#333 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/333>`__)
+- Parse the TELTRACK, TTBLTRCK, and TELSCAN header keys for POLCAL data with the new `TaskMostFrequentBud`. These keys are
+  only used to populate headers in dataset extras and there are some cases where the input data have multiple values in these
+  keys. While technically "incorrect", this doesn't affect any aspect of the pipeline or L1 data. In this case we log the
+  situation and move on. Parse HLS_VERS the same way for all dataset extra input data. (`#333 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/333>`__)
+
+
+Misc
+----
+
+- Only parse HLS_VERS header information for DARK, SOLAR_GAIN, and POLCAL task types. This key is only used for dataset
+  extra headers and these are the task types associated with dataset extras. (`#333 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/333>`__)
+
+
 v14.1.1 (2026-05-08)
 ====================
 

{dkist_processing_common-14.1.1 → dkist_processing_common-14.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 14.1.1
+Version: 14.2.0
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-14.1.1 → dkist_processing_common-14.2.0}/dkist_processing_common/parsers/unique_bud.py

@@ -1,13 +1,18 @@
 """Pre-made flower that reads a single header key from all files and raises a ValueError if it is not unique."""
 
+import logging
+from collections import Counter
 from enum import StrEnum
 from typing import Callable
 
+from dkist_processing_common.models.flower_pot import ListStem
 from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 
+logger = logging.getLogger(__name__)
+
 
 class UniqueBud(SetStem):
     """
@@ -100,3 +105,76 @@ class TaskUniqueBud(UniqueBud):
             return super().setter(fits_obj)
 
         return SpilledDirt
+
+
+class TaskMostFrequentBud(ListStem):
+    """
+    Bud that ingests values in a task-aware way and logs a warning if the set of values is not singular.
+
+    Unlike `TaskUniqueBud` this Bud will not raise an Error if there are more than one value; instead it will
+    return the most-common value.
+
+    This Bud is useful for cases where we expect the set of input values to contain a single value, but don't want to
+    stop processing if it isn't.
+
+    Parameters
+    ----------
+    constant_name
+        The name for the constant to be defined
+
+    metadata_key
+        The metadata key associated with the constant
+
+    ip_task_types
+        Only consider objects whose parsed header IP task type matches a string in this list
+
+    task_type_parsing_function
+        The function used to convert a header into an IP task type
+    """
+
+    def __init__(
+        self,
+        constant_name: str,
+        metadata_key: str | StrEnum,
+        ip_task_types: str | list[str],
+        task_type_parsing_function: Callable = passthrough_header_ip_task,
+    ):
+        super().__init__(stem_name=constant_name)
+
+        if isinstance(metadata_key, StrEnum):
+            metadata_key = metadata_key.name
+        self.metadata_key = metadata_key
+
+        if isinstance(ip_task_types, str):
+            ip_task_types = [ip_task_types]
+        self.ip_task_types = [task.casefold() for task in ip_task_types]
+
+        self.parsing_function = task_type_parsing_function
+
+    def setter(self, fits_obj: L0FitsAccess):
+        """Ingest an object only if its parsed IP task type matches what's desired."""
+        task = self.parsing_function(fits_obj)
+
+        if task.casefold() in self.ip_task_types:
+            return getattr(fits_obj, self.metadata_key)
+
+        return SpilledDirt
+
+    def getter(self):
+        """
+        Check the list of values, warn if the set of values is non-singular, and return the most-common value.
+
+        Returns
+        -------
+        Either the single or most-common value in `self.value_list`.
+        """
+        if (num_values := len(set(self.value_list))) > 1:
+            counts = Counter(self.value_list)
+            most_common_value = counts.most_common(1)[0][0]
+            logger.info(
+                f"Found {num_values} unique values for {self.stem_name}. "
+                f"Distribution is {counts}"
+            )
+            return most_common_value
+
+        return self.value_list[0]

{dkist_processing_common-14.1.1 → dkist_processing_common-14.2.0}/dkist_processing_common/tasks/parse_l0_input_data.py

@@ -51,6 +51,7 @@ from dkist_processing_common.parsers.time import TaskDateBeginBud
 from dkist_processing_common.parsers.time import TaskExposureTimesBud
 from dkist_processing_common.parsers.time import TaskReadoutExpTimesBud
 from dkist_processing_common.parsers.time import VarianceCadenceBud
+from dkist_processing_common.parsers.unique_bud import TaskMostFrequentBud
 from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
 from dkist_processing_common.parsers.unique_bud import UniqueBud
 from dkist_processing_common.tasks.base import WorkflowTaskBase
@@ -87,9 +88,11 @@ def dataset_extra_bud_factory() -> list[S]:
         UniqueBud(
             constant_name=BudName.software_binning_y, metadata_key=MetadataKey.software_binning_y
         ),
-        UniqueBud(
+        TaskMostFrequentBud(
             constant_name=BudName.hls_version,
             metadata_key=MetadataKey.hls_version,
+            ip_task_types=[TaskName.dark, TaskName.solar_gain, TaskName.polcal],
+            task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
         TaskContributingObservingProgramExecutionIdsBud(
             constant_name=BudName.dark_observing_program_execution_ids,
@@ -127,7 +130,7 @@ def dataset_extra_bud_factory() -> list[S]:
             ip_task_types=TaskName.solar_gain,
             task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
-        TaskUniqueBud(
+        TaskMostFrequentBud(
             constant_name=BudName.polcal_telescope_tracking_mode,
             metadata_key=MetadataKey.telescope_tracking_mode,
             ip_task_types=TaskName.polcal,
@@ -138,7 +141,7 @@ def dataset_extra_bud_factory() -> list[S]:
             ip_task_types=TaskName.solar_gain,
             task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
-        TaskUniqueBud(
+        TaskMostFrequentBud(
             constant_name=BudName.polcal_coude_table_tracking_mode,
             metadata_key=MetadataKey.coude_table_tracking_mode,
             ip_task_types=TaskName.polcal,
@@ -149,7 +152,7 @@ def dataset_extra_bud_factory() -> list[S]:
             ip_task_types=TaskName.solar_gain,
             task_type_parsing_function=parse_header_ip_task_with_gains,
         ),
-        TaskUniqueBud(
+        TaskMostFrequentBud(
             constant_name=BudName.polcal_telescope_scanning_mode,
             metadata_key=MetadataKey.telescope_scanning_mode,
             ip_task_types=TaskName.polcal,

{dkist_processing_common-14.1.1 → dkist_processing_common-14.2.0}/dkist_processing_common/tests/test_stems.py

@@ -1,4 +1,5 @@
 import collections
+import logging
 from enum import StrEnum
 from itertools import chain
 from typing import Generator
@@ -50,6 +51,7 @@ from dkist_processing_common.parsers.time import TaskExposureTimesBud
 from dkist_processing_common.parsers.time import TaskReadoutExpTimesBud
 from dkist_processing_common.parsers.time import TaskRoundTimeBudBase
 from dkist_processing_common.parsers.time import VarianceCadenceBud
+from dkist_processing_common.parsers.unique_bud import TaskMostFrequentBud
 from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
 from dkist_processing_common.parsers.unique_bud import UniqueBud
 from dkist_processing_common.parsers.wavelength import ObserveWavelengthBud
@@ -321,6 +323,30 @@ def bad_header_objs() -> Generator[FitsReader, None, None]:
                 "LINEWAV": 1.0,
             }
         ),
+        "thing5": fits.header.Header(
+            {
+                "id_key": 1,
+                "constant": 6.28,
+                "near": 1.76,
+                "DKIST004": "observe",
+                "DSPSREPS": 2,
+                "DSPSNUM": 2,
+                "DATE-OBS": "2022-06-17T22:00:03.000",
+                "LINEWAV": 1.0,
+            }
+        ),
+        "thing6": fits.header.Header(
+            {
+                "id_key": 1,
+                "constant": 6.66,
+                "near": 1.76,
+                "DKIST004": "dark",
+                "DSPSREPS": 2,
+                "DSPSNUM": 2,
+                "DATE-OBS": "2022-06-17T22:00:03.000",
+                "LINEWAV": 1.0,
+            }
+        ),
     }
     return (
         FitsReader(header=fits.Header(header), name=path) for path, header in bad_headers.items()
@@ -476,6 +502,60 @@ def test_task_unique_bud_non_unique_inputs(bad_header_objs, ip_task_type):
         _ = bud.bud
 
 
+@pytest.mark.parametrize(
+    "ip_task_type",
+    [
+        pytest.param("observe", id="single_task_type"),
+        pytest.param(["observe", "gain"], id="task_type_list"),
+    ],
+)
+def test_task_most_frequent_bud(basic_header_objs, ip_task_type, caplog):
+    """
+    Given: A set of headers with a constant value header key
+    When: Ingesting headers with a TaskMostFrequentBud and asking for the value
+    Then: The bud's value is the header constant value
+    """
+    bud_obj = TaskMostFrequentBud(
+        constant_name="proposal", metadata_key="proposal_id", ip_task_types=ip_task_type
+    )
+    assert bud_obj.stem_name == "proposal"
+    for fo in basic_header_objs:
+        key = fo.name
+        bud_obj.update(key, fo)
+
+    with caplog.at_level(logging.INFO):
+        assert bud_obj.bud.value == "proposal_id_1"
+        assert caplog.text == ""
+
+
+@pytest.mark.parametrize(
+    "ip_task_type, expected_value",
+    [
+        pytest.param("observe", 6.28, id="single_task_type"),
+        pytest.param(["dark", "gain"], 6.66, id="task_type_list"),
+    ],
+)
+def test_task_most_frequent_bud_non_unique_inputs(
+    bad_header_objs, ip_task_type, expected_value, caplog
+):
+    """
+    Given: A set of headers with a non-constant header key that is expected to be constant
+    When: Ingesting headers with a TaskMostFrequentBud and asking for the value
+    Then: The most-frequent value is returned and the correct warning is logged
+    """
+    bud_obj = TaskMostFrequentBud(
+        constant_name="constant", metadata_key="constant_thing", ip_task_types=ip_task_type
+    )
+    assert bud_obj.stem_name == "constant"
+    for fo in bad_header_objs:
+        key = fo.name
+        bud_obj.update(key, fo)
+
+    with caplog.at_level(logging.INFO):
+        assert bud_obj.bud.value == expected_value
+        assert "Found 2 unique values for constant" in caplog.text
+
+
 def test_single_value_single_key_flower(basic_header_objs):
     """
     Given: A set of filepaths and associated headers with a single key that has a limited set of values

{dkist_processing_common-14.1.1 → dkist_processing_common-14.2.0}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 14.1.1
+Version: 14.2.0
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause