dkist-processing-common 12.10.0__tar.gz → 13.0.0__tar.gz

{dkist_processing_common-12.10.0 → dkist_processing_common-13.0.0}/CHANGELOG.rst

@@ -1,3 +1,20 @@
+v13.0.0 (2026-04-03)
+====================
+
+Bugfixes
+--------
+
+- Update `*CadenceBuds` to correctly account for polarimetric data. Previously, the potentially different cadence
+  between modulation states was not considered when computing the cadence of L0 observations. (`#310 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/310>`__)
+
+
+Misc
+----
+
+- *All* children of `L0FitsAccess` are now required to define the `modulator_state` attribute. If an instrument is not
+  polarimetric, setting this attribute to -1 is recommended. (`#310 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/310>`__)
+
+
 v12.10.0 (2026-04-02)
 =====================
 

{dkist_processing_common-12.10.0 → dkist_processing_common-13.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 12.10.0
+Version: 13.0.0
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-12.10.0 → dkist_processing_common-13.0.0}/dkist_processing_common/parsers/task.py

@@ -13,7 +13,7 @@ from dkist_processing_common.parsers.single_value_single_key_flower import (
 )
 
 
-def passthrough_header_ip_task(fits_obj: Type[FitsAccessBase]) -> str:
+def passthrough_header_ip_task(fits_obj: FitsAccessBase) -> str:
     """
     Simply read the IP task directly from the header.
 

{dkist_processing_common-12.10.0 → dkist_processing_common-13.0.0}/dkist_processing_common/parsers/time.py

@@ -1,8 +1,10 @@
 """Time parser."""
 
+from collections import defaultdict
 from datetime import datetime
 from datetime import timezone
 from enum import StrEnum
+from typing import Any
 from typing import Callable
 from typing import Type
 
@@ -110,83 +112,134 @@ class TaskDatetimeBudBase(ListStem):
         return tuple(sorted(self.value_list))
 
 
-class CadenceBudBase(TaskDatetimeBudBase):
-    """Base class for all Cadence Buds."""
+class CadenceBudBase(ListStem):
+    """
+    Base class for all Cadence Buds.
+
+    Despite subclassing `ListStem`, this class does not use the `list` storage provided by that class. Instead, a dictionary
+    mapping modulation state to a list of times is used, which allows us to compute cadence values properly for polarimetric
+    data.
+
+    Two functions must be provided: one for reducing all timestamps for a given modstate, and one for reducing the individual
+    modstate values into the final cadence value. In other words, we first compute the requested "cadence" value individually
+    for each modstate, and then combine the value for all mostates into a single value that will be used for the constant.
+    """
 
-    def __init__(self, constant_name: str):
+    def __init__(
+        self,
+        constant_name: str,
+        diff_reducing_function: Callable[[np.ndarray], np.floating[Any]],
+        modstate_reducing_function: Callable[[list[np.floating[Any]]], np.floating[Any]],
+    ):
         super().__init__(
             stem_name=constant_name,
-            metadata_key=MetadataKey.time_obs,
-            ip_task_types=TaskName.observe,
         )
+        self.header_parsing_function = passthrough_header_ip_task
+        self.time_objs_metadata_key = MetadataKey.time_obs.name
+        self.ip_task_type = TaskName.observe.value.casefold()
+        self.mapping = defaultdict(list)
+        self.diff_reduce_fcn = diff_reducing_function
+        self.mod_reduce_fcn = modstate_reducing_function
+
+    def setter(self, fits_obj: L0FitsAccess) -> str | Type[SpilledDirt]:
+        """
+        Ingest a single `FitsAccess` object.
 
+        The object's timestamp is appended to the list of values associated with the object's modstate.
+        """
+        if self.header_parsing_function(fits_obj).casefold() == self.ip_task_type:
+            modstate = fits_obj.modulator_state
 
-class AverageCadenceBud(CadenceBudBase):
-    """Class for the average cadence Bud."""
+            timestamp = (
+                datetime.fromisoformat(getattr(fits_obj, self.time_objs_metadata_key))
+                .replace(tzinfo=timezone.utc)
+                .timestamp()
+            )
+            self.mapping[modstate].append(timestamp)
 
-    def __init__(self):
-        super().__init__(constant_name=BudName.average_cadence)
+            # Return values are important here; We have to return *something* if we're recording data we want to use.
+            # This string is that.
+            return "not used"
+
+        # And we have to return `SpilledDirt` if we didn't ingest this fits object
+        return SpilledDirt
 
-    def getter(self) -> np.float64:
+    def getter(self) -> np.floating[Any]:
         """
-        Return the mean cadence between frames.
+        Compute the cadence logic defined by the diff and modstate reducing functions.
 
-        Returns
-        -------
-        The mean value of the cadences of the input frames
+        For each modstate:
+          #. Sort the individual timestamps
+          #. Compute the `~numpy.diff` of this sorted list
+          #. Apply the ``diff_reduce_fcn`` to this diff array
+
+        Finally, apply the ``mod_reduce_fcn`` to the above results for all modstates to produce the final answer.
         """
-        return np.mean(np.diff(super().getter()))
+        per_modstate_values = [
+            self.diff_reduce_fcn(np.diff(sorted(time_list))) for time_list in self.mapping.values()
+        ]
+        final_value = self.mod_reduce_fcn(per_modstate_values)
+        return final_value
 
 
-class MaximumCadenceBud(CadenceBudBase):
-    """Class for the maximum cadence bud."""
+class AverageCadenceBud(CadenceBudBase):
+    """
+    Class for the average cadence Bud.
+
+    The final value is the average across modstates of the average time difference within each modstate.
+    """
 
     def __init__(self):
-        super().__init__(constant_name=BudName.maximum_cadence)
+        super().__init__(
+            constant_name=BudName.average_cadence,
+            diff_reducing_function=np.mean,
+            modstate_reducing_function=np.mean,
+        )
 
-    def getter(self) -> np.float64:
-        """
-        Return the maximum cadence between frames.
-
-        Returns
-        -------
-        The maximum cadence between frames
-        """
-        return np.max(np.diff(super().getter()))
 
+class MaximumCadenceBud(CadenceBudBase):
+    """
+    Class for the maximum cadence bud.
 
-class MinimumCadenceBud(CadenceBudBase):
-    """Class for the minimum cadence bud."""
+    The final value is the max across modstates of the max time difference within each modstate.
+    """
 
     def __init__(self):
-        super().__init__(constant_name=BudName.minimum_cadence)
-
-    def getter(self) -> np.float64:
-        """
-        Return the minimum cadence between frames.
+        super().__init__(
+            constant_name=BudName.maximum_cadence,
+            diff_reducing_function=np.max,
+            modstate_reducing_function=np.max,
+        )
 
-        Returns
-        -------
-        The minimum cadence between frames
-        """
-        return np.min(np.diff(super().getter()))
 
+class MinimumCadenceBud(CadenceBudBase):
+    """
+    Class for the minimum cadence bud.
 
-class VarianceCadenceBud(CadenceBudBase):
-    """Class for the variance cadence Bud."""
+    The final value is the minimum across modstates of the minimum time difference within each modstate.
+    """
 
     def __init__(self):
-        super().__init__(constant_name=BudName.variance_cadence)
+        super().__init__(
+            constant_name=BudName.minimum_cadence,
+            diff_reducing_function=np.min,
+            modstate_reducing_function=np.min,
+        )
 
-    def getter(self) -> np.float64:
-        """
-        Return the cadence variance between frames.
 
-        Returns
-        -------
-        Return the variance of the cadences over the input frames
-        """
-        return np.var(np.diff(super().getter()))
+class VarianceCadenceBud(CadenceBudBase):
+    """
+    Class for the variance cadence Bud.
+
+    The final value is the average across modstates of the variance time difference within each modstate.
+    """
+
+    def __init__(self):
+        super().__init__(
+            constant_name=BudName.variance_cadence,
+            diff_reducing_function=np.var,
+            modstate_reducing_function=np.mean,
+        )
 
 
 class TaskDateBeginBud(TaskDatetimeBudBase):

{dkist_processing_common-12.10.0 → dkist_processing_common-13.0.0}/dkist_processing_common/tests/test_stems.py

@@ -1,6 +1,7 @@
 import collections
 from enum import StrEnum
 from itertools import chain
+from typing import Generator
 
 import pytest
 from astropy.io import fits
@@ -20,6 +21,7 @@ from dkist_processing_common.parsers.dsps_repeat import TotalDspsRepeatsBud
 from dkist_processing_common.parsers.experiment_id_bud import ContributingExperimentIdsBud
 from dkist_processing_common.parsers.experiment_id_bud import ExperimentIdBud
 from dkist_processing_common.parsers.id_bud import TaskContributingIdsBud
+from dkist_processing_common.parsers.l0_fits_access import AbstractProperty
 from dkist_processing_common.parsers.lookup_bud import TaskTimeLookupBud
 from dkist_processing_common.parsers.lookup_bud import TimeLookupBud
 from dkist_processing_common.parsers.near_bud import NearFloatBud
@@ -75,9 +77,11 @@ class FitsReaderMetadataKey(StrEnum):
     gos_polarizer_status = "GOSPOL"
     wavelength = "LINEWAV"
     roundable_time = "RTIME"
+    modulator_state = "MODSTATE"
 
 
 class FitsReader(FitsAccessBase):
+
     def __init__(self, hdu, name):
         super().__init__(hdu, name)
         self.thing_id: int = self.header.get(FitsReaderMetadataKey.thing_id)
@@ -112,10 +116,17 @@ class FitsReader(FitsAccessBase):
         self.gos_polarizer_status: str = self.header.get(FitsReaderMetadataKey.gos_polarizer_status)
         self.wavelength: str = self.header.get(FitsReaderMetadataKey.wavelength)
         self.roundable_time: float = self.header.get(FitsReaderMetadataKey.roundable_time, 0.0)
+        self.modulator_state: int = -1
+
+
+class ModFitsReader(FitsReader):
+    def __init__(self, hdu, name):
+        super().__init__(hdu, name)
+        self.modulator_state: int = self.header.get(FitsReaderMetadataKey.modulator_state)
 
 
 @pytest.fixture()
-def basic_header_objs():
+def basic_header_objs() -> Generator[FitsReader, None, None]:
     header_dict = {
         "thing0": fits.header.Header(
             {
@@ -223,7 +234,7 @@ def basic_header_objs():
 
 
 @pytest.fixture
-def task_with_gains_header_objs():
+def task_with_gains_header_objs() -> Generator[FitsReader, None, None]:
     header_dict = {
         "lamp_gain": fits.header.Header({"DKIST004": "gain", "GOSLVL3": "lamp", "GOSLAMP": "on"}),
         "solar_gain": fits.header.Header({"DKIST004": "gain", "GOSLVL3": "clear"}),
@@ -238,7 +249,7 @@ def retarder_name():
 
 
 @pytest.fixture
-def task_with_polcal_header_objs(retarder_name):
+def task_with_polcal_header_objs(retarder_name) -> Generator[FitsReader, None, None]:
     header_dict = {
         "polcal_dark": fits.header.Header(
             {"DKIST004": "polcal", "GOSLVL0": "DarkShutter", "GOSPOL": "clear", "GOSRET": "clear"}
@@ -254,7 +265,7 @@ def task_with_polcal_header_objs(retarder_name):
 
 
 @pytest.fixture()
-def bad_header_objs():
+def bad_header_objs() -> Generator[FitsReader, None, None]:
     bad_headers = {
         "thing0": fits.header.Header(
             {
@@ -309,7 +320,7 @@ def bad_header_objs():
 
 
 @pytest.fixture
-def bad_polcal_header_objs():
+def bad_polcal_header_objs() -> Generator[FitsReader, None, None]:
     # I.e., GOSRET has multiple values
     header_dict = {
         "thing1": fits.header.Header({"DKIST004": "polcal", "GOSRET": "clear"}),
@@ -319,6 +330,52 @@ def bad_polcal_header_objs():
     return (FitsReader.from_header(header, name=path) for path, header in header_dict.items())
 
 
+@pytest.fixture
+def modulated_cadence_header_objs() -> Generator[FitsReader, None, None]:
+    header_dict = {
+        "thing1": fits.Header(
+            {"MODSTATE": 1, "DATE-OBS": "2000-01-01T00:00:00", "DKIST004": "observe"}
+        ),
+        "thing2": fits.Header(
+            {"MODSTATE": 1, "DATE-OBS": "2000-01-01T00:00:10", "DKIST004": "observe"}
+        ),
+        "thing3": fits.Header(
+            {"MODSTATE": 1, "DATE-OBS": "2000-01-01T00:00:22", "DKIST004": "observe"}
+        ),
+        "thing4": fits.Header(
+            {"MODSTATE": 2, "DATE-OBS": "2000-01-02T00:00:00", "DKIST004": "observe"}
+        ),
+        "thing5": fits.Header(
+            {"MODSTATE": 2, "DATE-OBS": "2000-01-02T00:00:05", "DKIST004": "observe"}
+        ),
+        "thing6": fits.Header(
+            {"MODSTATE": 2, "DATE-OBS": "2000-01-02T00:00:11", "DKIST004": "observe"}
+        ),
+        "thing7": fits.Header(
+            {"MODSTATE": 3, "DATE-OBS": "2000-01-03T00:00:00", "DKIST004": "observe"}
+        ),
+        "thing8": fits.Header(
+            {"MODSTATE": 3, "DATE-OBS": "2000-01-03T00:00:03", "DKIST004": "observe"}
+        ),
+        "thing9": fits.Header(
+            {"MODSTATE": 3, "DATE-OBS": "2000-01-03T00:00:07", "DKIST004": "observe"}
+        ),
+        "thing10": fits.Header(
+            {"MODSTATE": 3, "DATE-OBS": "2000-12-31T00:00:00", "DKIST004": "gain"}
+        ),
+    }
+    return (ModFitsReader.from_header(header, name=path) for path, header in header_dict.items())
+
+
+@pytest.fixture
+def non_observe_header_objs() -> Generator[FitsReader, None, None]:
+    header_dict = {
+        "thing1": fits.Header({"DKIST004": "gain"}),
+        "thing2": fits.Header({"DKIST004": "dark"}),
+    }
+    return (ModFitsReader.from_header(header, name=path) for path, header in header_dict.items())
+
+
 def test_unique_bud(basic_header_objs):
     """
     Given: A set of headers with a constant value header key
@@ -874,7 +931,7 @@ def test_dsps_flower(basic_header_objs):
     assert petals[1].keys == ["thing3"]
 
 
-def test_average_cadence_bud(basic_header_objs):
+def test_average_cadence_bud(basic_header_objs, modulated_cadence_header_objs):
     """
     Given: A set of filepaths and associated headers with DATE-OBS keywords
     When: Ingesting with the AverageCadenceBud
@@ -889,8 +946,15 @@ def test_average_cadence_bud(basic_header_objs):
     # Because there are 3 observe frames in `basic_header_objs` spaced 1, and 2 seconds apart.
     assert bud_obj.bud.value == 1.5
 
+    pol_bud_obj = AverageCadenceBud()
+    for fo in modulated_cadence_header_objs:
+        key = fo.name
+        pol_bud_obj.update(key, fo)
+
+    assert pol_bud_obj.bud.value == 20 / 3
+
 
-def test_max_cadence_bud(basic_header_objs):
+def test_max_cadence_bud(basic_header_objs, modulated_cadence_header_objs):
     """
     Given: A set of filepaths and associated headers with DATE-OBS keywords
     When: Ingesting with the MaxCadenceBud
@@ -905,8 +969,15 @@ def test_max_cadence_bud(basic_header_objs):
     # Because there are 3 observe frames in `basic_header_objs` spaced 1, and 2 seconds apart.
     assert bud_obj.bud.value == 2
 
+    pol_bud_obj = MaximumCadenceBud()
+    for fo in modulated_cadence_header_objs:
+        key = fo.name
+        pol_bud_obj.update(key, fo)
+
+    assert pol_bud_obj.bud.value == 12
+
 
-def test_minimum_cadence_bud(basic_header_objs):
+def test_minimum_cadence_bud(basic_header_objs, modulated_cadence_header_objs):
     """
     Given: A set of filepaths and associated headers with DATE-OBS keywords
     When: Ingesting with the MinimumCadenceBud
@@ -921,8 +992,15 @@ def test_minimum_cadence_bud(basic_header_objs):
     # Because there are 3 observe frames in `basic_header_objs` spaced 1, and 2 seconds apart.
     assert bud_obj.bud.value == 1
 
+    pol_bud_obj = MinimumCadenceBud()
+    for fo in modulated_cadence_header_objs:
+        key = fo.name
+        pol_bud_obj.update(key, fo)
+
+    assert pol_bud_obj.bud.value == 3
 
-def test_variance_cadence_bud(basic_header_objs):
+
+def test_variance_cadence_bud(basic_header_objs, modulated_cadence_header_objs):
     """
     Given: A set of filepaths and associated headers with DATE-OBS keywords
     When: Ingesting with the VarianceCadenceBud
@@ -937,6 +1015,37 @@ def test_variance_cadence_bud(basic_header_objs):
     # Because there are 3 observe frames in `basic_header_objs` spaced 1, and 2 seconds apart.
     assert bud_obj.bud.value == 0.25
 
+    pol_bud_obj = VarianceCadenceBud()
+    for fo in modulated_cadence_header_objs:
+        key = fo.name
+        pol_bud_obj.update(key, fo)
+
+    assert pol_bud_obj.bud.value == 0.5
+
+
+def test_cadence_buds_no_observes(non_observe_header_objs):
+    """
+    Given: A set of filepaths and headers that contain no "observe" frames
+    When: Ingesting these headers with the *CadenceBuds and checking if there are values to be picked
+    Then: None of the Buds "can_be_picked"
+    """
+    avg_bud = AverageCadenceBud()
+    min_bud = MinimumCadenceBud()
+    max_bud = MaximumCadenceBud()
+    var_bud = VarianceCadenceBud()
+
+    for fo in non_observe_header_objs:
+        key = fo.name
+        avg_bud.update(key, fo)
+        min_bud.update(key, fo)
+        max_bud.update(key, fo)
+        var_bud.update(key, fo)
+
+    assert not avg_bud.can_be_picked
+    assert not min_bud.can_be_picked
+    assert not max_bud.can_be_picked
+    assert not var_bud.can_be_picked
+
 
 def test_task_date_begin_bud(basic_header_objs):
     """

{dkist_processing_common-12.10.0 → dkist_processing_common-13.0.0}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 12.10.0
+Version: 13.0.0
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause