dkist-processing-common 12.0.0rc7__tar.gz → 12.1.0__tar.gz

{dkist_processing_common-12.0.0rc7 → dkist_processing_common-12.1.0}/.readthedocs.yml

@@ -2,7 +2,7 @@ version: 2
 build:
   os: ubuntu-20.04
   tools:
-    python: "3.12"
+    python: "3.13"
   apt_packages:
     - libopenjp2-7
     - graphviz

{dkist_processing_common-12.0.0rc7 → dkist_processing_common-12.1.0}/CHANGELOG.rst

@@ -1,3 +1,38 @@
+v12.1.0 (2026-01-26)
+====================
+
+Features
+--------
+
+- Add `ListStem` base class for huge speedup in cases where the keys don't matter and the `getter` logic only depends on the
+  list of values computed by `setter`. This is the case for most (all?) "Buds". (`#282 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/282>`__)
+- Add `SetStem` base class that has all the benefits of `ListStem` but also gets a speedup by storing values in a `set` for
+  cases where repeated values don't need to be tracked. (`#282 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/282>`__)
+- Speed up parsing of the `*CadenceBud`, `TaskDateBeginBud`, and `[Task]NearFloatBud` by basing these buds on `ListStem`. (`#284 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/284>`__)
+- Speed up `NumCSStepBud`, `[Task]UniqueBud`, `[Task]ContributingIdsBud`, and `TaskRoundTimeBudBase` parsing by basing
+  these buds on `SetStem`. (`#285 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/285>`__)
+- Speed up `CSStepFlower` parsing by using an internal set to keep track of the unique `CSStep` objects. This removes the
+  need to compute the unique set when computing the tag for each file. (`#286 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/286>`__)
+
+
+Misc
+----
+
+- Speed up the reading of INPUT files in Parse tasks by turning off image decompression and checksum checks. (`#280 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/280>`__)
+- Update `RetarderNameBud` to drop "clear" values (i.e., the retarder is out of the beam) in the `setter` instead of the `getter`.
+  This brings it in line with standard Bud-practice. (`#285 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/285>`__)
+- Convert the TimeLookupBud to be a SetStem constant. (`#287 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/287>`__)
+
+
+v12.0.0 (2026-01-22)
+====================
+
+Misc
+----
+
+- Upgrade dkist-processing-core to 7.0.0 which includes an upgrade of Airflow to 3.1.5 and python >= 3.13. (`#278 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/278>`__)
+
+
 v11.9.3 (2026-01-07)
 ====================
 

{dkist_processing_common-12.0.0rc7 → dkist_processing_common-12.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 12.0.0rc7
+Version: 12.1.0
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -17,7 +17,7 @@ Requires-Dist: asdf<4.0.0,>=3.5.0
 Requires-Dist: astropy>=7.0.0
 Requires-Dist: dkist-fits-specifications<5.0,>=4.0.0
 Requires-Dist: dkist-header-validator<6.0,>=5.0.0
-Requires-Dist: dkist-processing-core==7.0.0rc9
+Requires-Dist: dkist-processing-core==7.0.1
 Requires-Dist: dkist-processing-pac<4.0,>=3.1
 Requires-Dist: dkist-service-configuration<5.0,>=4.2.0
 Requires-Dist: dkist-spectral-lines<4.0,>=3.0.0

{dkist_processing_common-12.0.0rc7 → dkist_processing_common-12.1.0}/dkist_processing_common/codecs/fits.py

@@ -31,15 +31,30 @@ def fits_hdulist_encoder(hdu_list: fits.HDUList) -> bytes:
     return iobase_encoder(file_obj)
 
 
-def fits_hdu_decoder(path: Path, hdu: int | None = None) -> fits.PrimaryHDU | fits.CompImageHDU:
+def fits_hdu_decoder(
+    path: Path,
+    hdu: int | None = None,
+    checksum: bool = True,
+    disable_image_compression: bool = False,
+) -> fits.PrimaryHDU | fits.CompImageHDU:
     """Read a Path with `fits` to produce an `HDUList`."""
-    hdu_list = fits.open(path, checksum=True)
+    hdu_list = fits.open(
+        path, checksum=checksum, disable_image_compression=disable_image_compression
+    )
     return _extract_hdu(hdu_list, hdu)
 
 
-def fits_array_decoder(path: Path, hdu: int | None = None, auto_squeeze: bool = True) -> np.ndarray:
+def fits_array_decoder(
+    path: Path,
+    hdu: int | None = None,
+    auto_squeeze: bool = True,
+    checksum: bool = True,
+    disable_image_compression: bool = False,
+) -> np.ndarray:
     """Read a Path with `fits` and return the `.data` property."""
-    hdu = fits_hdu_decoder(path, hdu=hdu)
+    hdu = fits_hdu_decoder(
+        path, hdu=hdu, checksum=checksum, disable_image_compression=disable_image_compression
+    )
     data = hdu.data
 
     # This conditional is explicitly to catch summit data with a dummy first axis for WCS
@@ -50,10 +65,16 @@ def fits_array_decoder(path: Path, hdu: int | None = None, auto_squeeze: bool =
 
 
 def fits_access_decoder(
-    path: Path, fits_access_class: Type[FitsAccessBase], **fits_access_kwargs
+    path: Path,
+    fits_access_class: Type[FitsAccessBase],
+    checksum: bool = True,
+    disable_image_compression: bool = False,
+    **fits_access_kwargs,
 ) -> FitsAccessBase:
     """Read a Path with `fits` and ingest into a `FitsAccessBase`-type object."""
-    hdu = fits_hdu_decoder(path)
+    hdu = fits_hdu_decoder(
+        path, checksum=checksum, disable_image_compression=disable_image_compression
+    )
     return fits_access_class(hdu=hdu, name=str(path), **fits_access_kwargs)
 
 

{dkist_processing_common-12.0.0rc7 → dkist_processing_common-12.1.0}/dkist_processing_common/models/constants.py

@@ -333,8 +333,8 @@ class ConstantsBase:
     def dark_num_raw_frames_per_fpa(self) -> dict[float, list]:
         """Return the dictionary of exposure times to number of raw frames per fpa."""
         raw_return = self._db_dict[BudName.dark_num_raw_frames_per_fpa]
-        # convert nested lists to dictionary
-        return {k: v for k, v in raw_return}
+        # convert JSON string keys back to float
+        return {float(k): v for k, v in raw_return.items()}
 
     @property
     def solar_gain_num_raw_frames_per_fpa(self) -> int:

dkist_processing_common-12.1.0/dkist_processing_common/models/flower_pot.py

@@ -0,0 +1,416 @@
+"""
+Framework for grouping multiple keys and values with arbitrary logic.
+
+The key components are:
+    **Stem:** ABC for groupings that depend on both the key and (maybe) value. Subgroups (Petals) are implied but not enforced.
+
+    **ListStem:** ABC for groups that depend on value only. More limited, but faster than `Stem` for cases where the keys don't matter.
+
+    **SetStem:** ABC for groups that depend on value only and the values are well represented by a `set`. Even more limited, but faster than `Stem` for cases where the keys don't matter.
+
+    **FlowerPot:** Container for Stem children (Flowers)
+"""
+
+from __future__ import annotations
+
+from abc import ABC
+from abc import abstractmethod
+from collections import defaultdict
+from collections.abc import Hashable
+from typing import Any
+
+
+class FlowerPot:
+    """Base class to hold multiple sets (Stems) of key, value pairs."""
+
+    def __init__(self):
+        self.stems: list[Stem] = list()
+
+    def __iter__(self):
+        return self.stems.__iter__()
+
+    def __len__(self):
+        return self.stems.__len__()
+
+    def __getitem__(self, item):
+        return self.stems.__getitem__(item)
+
+    def add_dirt(self, key: Hashable, value: Any) -> None:
+        """
+        Send key and value through all Stems.
+
+        Parameters
+        ----------
+        key
+            The key
+        value
+            The value
+
+        Returns
+        -------
+        None
+        """
+        if not isinstance(key, Hashable):
+            raise TypeError(f"Type of key ({type(key)}) is not hashable")
+
+        for stem in self.stems:
+            stem.update(key, value)
+
+
+class SpilledDirt:
+    """
+    A custom class for when a Stem wants the FlowerPot to skip that particular key/value.
+
+    Exists because None, False, [], (), etc. etc. are all valid Stem return values
+    """
+
+
+class Thorn:
+    """
+    Custom class to indicate that a Bud's value should not be used.
+
+    I.e., don't pick this Bud because it's thorny. This exists for "Picky" Buds that merely perform a check and Error
+    on failure. If the check passes we don't need to do anything with the Bud's value and that value should be a Thorn.
+    """
+
+
+class Petal:
+    """
+    Base class to hold a single key, value pair.
+
+    Parameters
+    ----------
+    item
+        The key, value pair to be added
+    """
+
+    def __init__(self, item: tuple):
+        self.value = item[0]
+        self.keys = item[1]
+
+    def __repr__(self):
+        return f"Petal: {{{self.value}: {self.keys}}}"
+
+
+class Stem(ABC):
+    """
+    Base class for grouping keys via arbitrary logic on the total collection of keys and values.
+
+    Parameters
+    ----------
+    stem_name
+        The name to be associated with the stem
+    """
+
+    def __init__(self, stem_name: Any):
+        self.stem_name = stem_name
+        self.key_to_petal_dict: dict[Hashable, Hashable] = dict()
+
+        self._petal_cache: list[Petal] = []
+        self._need_to_generate_petals: bool = True
+
+    def update(self, key: Hashable, value: Any) -> None:
+        """
+        Ingest a single key/value pair.
+
+        Parameters
+        ----------
+        key
+            The key
+        value
+            The value
+
+        Returns
+        -------
+        None
+        """
+        result = self.setter(value)
+        if result is not SpilledDirt:
+            self.key_to_petal_dict[key] = result
+            self._need_to_generate_petals = True
+
+    @property
+    def petals(self) -> list[Petal]:
+        """Return subgroups and associated keys."""
+        if self._need_to_generate_petals:
+            self._generate_petal_list()
+
+        return self._petal_cache
+
+    @property
+    def can_be_picked(self) -> bool:
+        """
+        Return True if there are any values to be picked.
+
+        A `Stem` could have no values even after dirt is added if all of the results were `SpilledDirt`.
+        """
+        return len(self.petals) > 0
+
+    def _generate_petal_list(self) -> None:
+        """
+        Generate a list of petals.
+
+        Generating a petal list can be expensive because it involves inverting the `key_to_petal_dict`. To avoid doing
+        this every the `petals` property is accessed, this method produces a cached list of petals that is only
+        re-generated when the `key_to_petal_dict` has been updated.
+
+        Note: We can't make `petals` a `@cached_property` because that would make `petals` always tied to whatever the
+        state of this `Stem` was when `petals` was first accessed. And we can't use `lru_cache` because the object that
+        changes, `key_to_petal_dict`, is unhashable.
+        """
+        petal_to_key_dict = defaultdict(list)
+        for key in self.key_to_petal_dict.keys():
+            petal = self.getter(key)
+            petal_to_key_dict[petal].append(key)
+
+        self._petal_cache = [Petal(item) for item in petal_to_key_dict.items()]
+        self._need_to_generate_petals = False
+
+    @property
+    def bud(self) -> Petal:
+        """Just the first petal."""
+        bud = self.petals[0]
+        return bud
+
+    @abstractmethod
+    def setter(self, value: Any) -> Any:
+        """
+        Logic to apply to a single key/value pair on ingest.
+
+        Implemented in derived class.
+
+        Parameters
+        ----------
+        value
+            The value to be added
+
+        Returns
+        -------
+        Any
+        """
+        pass
+
+    @abstractmethod
+    def getter(self, key: Hashable) -> Hashable:
+        """
+        Logic to apply to all ingested values when picking the Stem.
+
+        Implemented in derived class.
+
+        Parameters
+        ----------
+        key
+            The key to return the value of
+        Returns
+        -------
+        The value
+        """
+        pass
+
+
+class ListStem(ABC):
+    """
+    Base class for collecting and applying logic to values in a `list` with a `Stem`-like interface.
+
+    Unlike the full `Stem`, this class does NOT retain information about the keys and thus does no grouping of keys based
+    on values. The direct consequence of this is that the `.petals` property is undefined and will raise an ``AttributeError``
+    if accessed. This also means there is no need to invert the `key_to_petal_dict` (because it doesn't exist), which,
+    in turn, means there is no need to run the `getter` for every key. The result is that the `bud` property only needs
+    one call to `getter`. Thus, the calculation of a single value derived from all values (i.e., `bud`) is much faster
+    than using a full `Stem`.
+
+    Parameters
+    ----------
+    stem_name
+        The name to be associated with the stem
+    """
+
+    def __init__(self, stem_name: Any):
+        self.stem_name = stem_name
+        self.value_list: list = list()
+        self._need_to_compute_bud_value: bool = True
+
+    def update(self, key: Any, value: Any) -> None:
+        """
+        Ingest a single key/value pair. Note that the ``key`` is not used.
+
+        Parameters
+        ----------
+        key
+            The key (unused)
+
+        value
+            The value
+
+        Returns
+        -------
+        None
+        """
+        result = self.setter(value)
+        if result is not SpilledDirt:
+            self.value_list.append(result)
+            self._need_to_compute_bud_value = True
+
+    @property
+    def petals(self) -> None:
+        """Raise an error because `ListStem` does not retain key information and therefore cannot group keys."""
+        raise AttributeError(
+            f"{self.__class__.__name__} subclasses ListStem and therefore does not define the `petals` property"
+        )
+
+    @property
+    def can_be_picked(self) -> bool:
+        """
+        Return True if there are any values to be picked.
+
+        A `Stem` could have no values even after dirt is added if all of the results were `SpilledDirt`.
+        """
+        return len(self.value_list) > 0
+
+    @property
+    def bud(self) -> Petal:
+        """Return the result of `getter` packaged in a `Petal` object."""
+        if self._need_to_compute_bud_value:
+            self._value_cache = self.getter()
+            self._need_to_compute_bud_value = False
+
+        return Petal((self._value_cache, "LISTSTEM_NOT_USED"))
+
+    @abstractmethod
+    def setter(self, value: Any) -> Any:
+        """
+        Logic to apply to a single value pair on ingest.
+
+        Implemented in derived class.
+
+        Parameters
+        ----------
+        value
+            The value to be added
+
+        Returns
+        -------
+        Any
+        """
+        pass
+
+    @abstractmethod
+    def getter(self) -> Any:
+        """
+        Logic to apply to all ingested values when computing the `bud`.
+
+        Implemented in derived class.
+
+        Returns
+        -------
+        The value of the bud
+        """
+        pass
+
+
+class SetStem(ABC):
+    """
+    Base class for collecting and applying logic to values in a `set` with a `Stem`-like interface.
+
+    Unlike the full `Stem`, this class does NOT retain information about the keys and thus does no grouping of keys based
+    on values. The direct consequence of this is that the `.petals` property is undefined and will raise an ``AttributeError``
+    if accessed. This also means there is no need to invert the `key_to_petal_dict` (because it doesn't exist), which,
+    in turn, means there is no need to run the `getter` for every key. The result is that the `bud` property only needs
+    one call to `getter`. Combined with the efficiency of storing values in a `set`, the calculation of a single value
+    derived from all values (i.e., `bud`) is much faster than using a full `Stem`.
+
+    .. Note::
+      The use of a `set` as the underlying storage mechanism means information regarding how many times a particular value
+      is present will be lost. It also means the return type of `setter` must be hashable. Both of these constraints can
+      be avoided by using `ListStem`, which still gets a significant speedup over `Stem` by dropping key information.
+
+
+    Parameters
+    ----------
+    stem_name
+        The name to be associated with the stem
+    """
+
+    def __init__(self, stem_name: Any):
+        self.stem_name = stem_name
+        self.value_set: set = set()
+        self._need_to_compute_bud_value: bool = True
+
+    def update(self, key: Any, value: Any) -> None:
+        """
+        Ingest a single key/value pair. Note that the ``key`` is not used.
+
+        Parameters
+        ----------
+        key
+            The key (unused)
+
+        value
+            The value
+
+        Returns
+        -------
+        None
+        """
+        result = self.setter(value)
+        if result is not SpilledDirt:
+            self.value_set.add(result)
+            self._need_to_compute_bud_value = True
+
+    @property
+    def petals(self) -> None:
+        """Raise an error because `SetStem` does not retain key information and therefore cannot group keys."""
+        raise AttributeError(
+            f"{self.__class__.__name__} subclasses SetStem and therefore does not define the `petals` property"
+        )
+
+    @property
+    def can_be_picked(self) -> bool:
+        """
+        Return True if there are any values to be picked.
+
+        A `Stem` could have no values even after dirt is added if all of the results were `SpilledDirt`.
+        """
+        return len(self.value_set) > 0
+
+    @property
+    def bud(self) -> Petal:
+        """Return the result of `getter` packaged in a `Petal` object."""
+        if self._need_to_compute_bud_value:
+            self._value_cache = self.getter()
+            self._need_to_compute_bud_value = False
+
+        return Petal((self._value_cache, "SETSTEM_NOT_USED"))
+
+    @abstractmethod
+    def setter(self, value: Any) -> Hashable:
+        """
+        Logic to apply to a single value pair on ingest.
+
+        Must return a Hashable object because the result will be stored in a `set`.
+
+        Implemented in derived class.
+
+        Parameters
+        ----------
+        value
+            The value to be added
+
+        Returns
+        -------
+        Any
+        """
+        pass
+
+    @abstractmethod
+    def getter(self) -> Any:
+        """
+        Logic to apply to all ingested values when computing the `bud`.
+
+        Implemented in derived class.
+
+        Returns
+        -------
+        The value of the bud
+        """
+        pass

{dkist_processing_common-12.0.0rc7 → dkist_processing_common-12.1.0}/dkist_processing_common/parsers/average_bud.py

@@ -30,8 +30,6 @@ class TaskAverageBud(TaskNearFloatBud):
         The function used to convert a header into an IP task type
     """
 
-    key_to_petal_dict: dict[str, float]
-
     def __init__(
         self,
         constant_name: str,

{dkist_processing_common-12.0.0rc7 → dkist_processing_common-12.1.0}/dkist_processing_common/parsers/cs_step.py

@@ -7,6 +7,7 @@ from datetime import timezone
 from typing import Type
 
 from dkist_processing_common.models.constants import BudName
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.tags import StemName
@@ -103,6 +104,7 @@ class CSStepFlower(Stem):
     def __init__(self, max_cs_step_time_sec: float):
         super().__init__(stem_name=StemName.cs_step)
         self.max_cs_step_time_sec = max_cs_step_time_sec
+        self.CS_step_set = set()
 
     def setter(self, fits_obj: L0FitsAccess) -> CSStep | Type[SpilledDirt]:
         """
@@ -119,7 +121,10 @@ class CSStepFlower(Stem):
         """
         if fits_obj.ip_task_type != "polcal":
             return SpilledDirt
-        return CSStep(fits_obj, max_cs_time_sec=self.max_cs_step_time_sec)
+
+        cs_step = CSStep(fits_obj, max_cs_time_sec=self.max_cs_step_time_sec)
+        self.CS_step_set.add(cs_step)
+        return cs_step
 
     def getter(self, key) -> str | float | int:
         """
@@ -133,11 +138,11 @@ class CSStepFlower(Stem):
         -------
         The cs step for the given key
         """
-        unique_steps = sorted(list(set(self.key_to_petal_dict.values())))
+        unique_steps = sorted(self.CS_step_set)
         return unique_steps.index(self.key_to_petal_dict[key])
 
 
-class NumCSStepBud(Stem):
+class NumCSStepBud(SetStem):
     """
     The total number of CS Steps present in a dataset.
 
@@ -168,17 +173,12 @@ class NumCSStepBud(Stem):
             return SpilledDirt
         return CSStep(fits_obj, max_cs_time_sec=self.max_cs_step_time_sec)
 
-    def getter(self, key) -> int:
+    def getter(self) -> int:
         """
         Return the number of CS Steps present.
 
-        Parameters
-        ----------
-        key
-            The input key
         Returns
         -------
         The number of cs steps associated with the key
         """
-        value_set = set(self.key_to_petal_dict.values())
-        return len(value_set)
+        return len(self.value_set)

{dkist_processing_common-12.0.0rc7 → dkist_processing_common-12.1.0}/dkist_processing_common/parsers/id_bud.py

@@ -4,13 +4,13 @@ from enum import StrEnum
 from typing import Callable
 from typing import Type
 
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
-from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 
 
-class ContributingIdsBud(Stem):
+class ContributingIdsBud(SetStem):
     """Base class for contributing ID buds."""
 
     def __init__(self, constant_name: str, metadata_key: str | StrEnum):
@@ -33,20 +33,15 @@ class ContributingIdsBud(Stem):
         """
         return getattr(fits_obj, self.metadata_key)
 
-    def getter(self, key) -> tuple[str, ...]:
+    def getter(self) -> tuple[str, ...]:
         """
         Get all ids seen for any type of frame.
 
-        Parameters
-        ----------
-        key
-            The input key
-
         Returns
         -------
         IDs from all types of frames
         """
-        return tuple(set(self.key_to_petal_dict.values()))
+        return tuple(self.value_set)
 
 
 class TaskContributingIdsBud(ContributingIdsBud):