dkist-processing-common 12.0.0rc7__py3-none-any.whl → 12.1.0__py3-none-any.whl

dkist_processing_common/codecs/fits.py

@@ -31,15 +31,30 @@ def fits_hdulist_encoder(hdu_list: fits.HDUList) -> bytes:
     return iobase_encoder(file_obj)
 
 
-def fits_hdu_decoder(path: Path, hdu: int | None = None) -> fits.PrimaryHDU | fits.CompImageHDU:
+def fits_hdu_decoder(
+    path: Path,
+    hdu: int | None = None,
+    checksum: bool = True,
+    disable_image_compression: bool = False,
+) -> fits.PrimaryHDU | fits.CompImageHDU:
     """Read a Path with `fits` to produce an `HDUList`."""
-    hdu_list = fits.open(path, checksum=True)
+    hdu_list = fits.open(
+        path, checksum=checksum, disable_image_compression=disable_image_compression
+    )
     return _extract_hdu(hdu_list, hdu)
 
 
-def fits_array_decoder(path: Path, hdu: int | None = None, auto_squeeze: bool = True) -> np.ndarray:
+def fits_array_decoder(
+    path: Path,
+    hdu: int | None = None,
+    auto_squeeze: bool = True,
+    checksum: bool = True,
+    disable_image_compression: bool = False,
+) -> np.ndarray:
     """Read a Path with `fits` and return the `.data` property."""
-    hdu = fits_hdu_decoder(path, hdu=hdu)
+    hdu = fits_hdu_decoder(
+        path, hdu=hdu, checksum=checksum, disable_image_compression=disable_image_compression
+    )
     data = hdu.data
 
     # This conditional is explicitly to catch summit data with a dummy first axis for WCS
@@ -50,10 +65,16 @@ def fits_array_decoder(path: Path, hdu: int | None = None, auto_squeeze: bool =
 
 
 def fits_access_decoder(
-    path: Path, fits_access_class: Type[FitsAccessBase], **fits_access_kwargs
+    path: Path,
+    fits_access_class: Type[FitsAccessBase],
+    checksum: bool = True,
+    disable_image_compression: bool = False,
+    **fits_access_kwargs,
 ) -> FitsAccessBase:
     """Read a Path with `fits` and ingest into a `FitsAccessBase`-type object."""
-    hdu = fits_hdu_decoder(path)
+    hdu = fits_hdu_decoder(
+        path, checksum=checksum, disable_image_compression=disable_image_compression
+    )
     return fits_access_class(hdu=hdu, name=str(path), **fits_access_kwargs)
 
 

dkist_processing_common/models/constants.py

@@ -333,8 +333,8 @@ class ConstantsBase:
     def dark_num_raw_frames_per_fpa(self) -> dict[float, list]:
         """Return the dictionary of exposure times to number of raw frames per fpa."""
         raw_return = self._db_dict[BudName.dark_num_raw_frames_per_fpa]
-        # convert nested lists to dictionary
-        return {k: v for k, v in raw_return}
+        # convert JSON string keys back to float
+        return {float(k): v for k, v in raw_return.items()}
 
     @property
     def solar_gain_num_raw_frames_per_fpa(self) -> int:

dkist_processing_common/models/flower_pot.py

@@ -1,10 +1,14 @@
 """
 Framework for grouping multiple keys and values with arbitrary logic.
 
-Defines:
-    Stem -> ABC for groupings that depend on both the key and (maybe) value. Subgroups (Petals) are implied but not enforced.
+The key components are:
+    **Stem:** ABC for groupings that depend on both the key and (maybe) value. Subgroups (Petals) are implied but not enforced.
 
-    FlowerPot -> Container for Stem children (Flowers)
+    **ListStem:** ABC for groups that depend on value only. More limited, but faster than `Stem` for cases where the keys don't matter.
+
+    **SetStem:** ABC for groups that depend on value only and the values are well represented by a `set`. Even more limited, but faster than `Stem` for cases where the keys don't matter.
+
+    **FlowerPot:** Container for Stem children (Flowers)
 """
 
 from __future__ import annotations
@@ -17,7 +21,7 @@ from typing import Any
 
 
 class FlowerPot:
-    """Base class to hold multiple sets (stems) of key, value pairs."""
+    """Base class to hold multiple sets (Stems) of key, value pairs."""
 
     def __init__(self):
         self.stems: list[Stem] = list()
@@ -55,9 +59,9 @@ class FlowerPot:
 
 class SpilledDirt:
     """
-    A custom class for when a Flower wants the FlowerPot to skip that particular key/value.
+    A custom class for when a Stem wants the FlowerPot to skip that particular key/value.
 
-    Exists because None, False, [], (), etc. etc. are all valid Flower return values
+    Exists because None, False, [], (), etc. etc. are all valid Stem return values
     """
 
 
@@ -90,7 +94,7 @@ class Petal:
 
 class Stem(ABC):
     """
-    Base group for grouping keys via arbitrary logic on the total collection of keys and values.
+    Base class for grouping keys via arbitrary logic on the total collection of keys and values.
 
     Parameters
     ----------
@@ -133,6 +137,15 @@ class Stem(ABC):
 
         return self._petal_cache
 
+    @property
+    def can_be_picked(self) -> bool:
+        """
+        Return True if there are any values to be picked.
+
+        A `Stem` could have no values even after dirt is added if all of the results were `SpilledDirt`.
+        """
+        return len(self.petals) > 0
+
     def _generate_petal_list(self) -> None:
         """
         Generate a list of petals.
@@ -146,7 +159,7 @@ class Stem(ABC):
         changes, `key_to_petal_dict`, is unhashable.
         """
         petal_to_key_dict = defaultdict(list)
-        for key, petal in self.key_to_petal_dict.items():
+        for key in self.key_to_petal_dict.keys():
             petal = self.getter(key)
             petal_to_key_dict[petal].append(key)
 
@@ -180,7 +193,7 @@ class Stem(ABC):
     @abstractmethod
     def getter(self, key: Hashable) -> Hashable:
         """
-        Logic to apply to all ingested values when picking the Flower.
+        Logic to apply to all ingested values when picking the Stem.
 
         Implemented in derived class.
 
@@ -193,3 +206,211 @@ class Stem(ABC):
         The value
         """
         pass
+
+
+class ListStem(ABC):
+    """
+    Base class for collecting and applying logic to values in a `list` with a `Stem`-like interface.
+
+    Unlike the full `Stem`, this class does NOT retain information about the keys and thus does no grouping of keys based
+    on values. The direct consequence of this is that the `.petals` property is undefined and will raise an ``AttributeError``
+    if accessed. This also means there is no need to invert the `key_to_petal_dict` (because it doesn't exist), which,
+    in turn, means there is no need to run the `getter` for every key. The result is that the `bud` property only needs
+    one call to `getter`. Thus, the calculation of a single value derived from all values (i.e., `bud`) is much faster
+    than using a full `Stem`.
+
+    Parameters
+    ----------
+    stem_name
+        The name to be associated with the stem
+    """
+
+    def __init__(self, stem_name: Any):
+        self.stem_name = stem_name
+        self.value_list: list = list()
+        self._need_to_compute_bud_value: bool = True
+
+    def update(self, key: Any, value: Any) -> None:
+        """
+        Ingest a single key/value pair. Note that the ``key`` is not used.
+
+        Parameters
+        ----------
+        key
+            The key (unused)
+
+        value
+            The value
+
+        Returns
+        -------
+        None
+        """
+        result = self.setter(value)
+        if result is not SpilledDirt:
+            self.value_list.append(result)
+            self._need_to_compute_bud_value = True
+
+    @property
+    def petals(self) -> None:
+        """Raise an error because `ListStem` does not retain key information and therefore cannot group keys."""
+        raise AttributeError(
+            f"{self.__class__.__name__} subclasses ListStem and therefore does not define the `petals` property"
+        )
+
+    @property
+    def can_be_picked(self) -> bool:
+        """
+        Return True if there are any values to be picked.
+
+        A `Stem` could have no values even after dirt is added if all of the results were `SpilledDirt`.
+        """
+        return len(self.value_list) > 0
+
+    @property
+    def bud(self) -> Petal:
+        """Return the result of `getter` packaged in a `Petal` object."""
+        if self._need_to_compute_bud_value:
+            self._value_cache = self.getter()
+            self._need_to_compute_bud_value = False
+
+        return Petal((self._value_cache, "LISTSTEM_NOT_USED"))
+
+    @abstractmethod
+    def setter(self, value: Any) -> Any:
+        """
+        Logic to apply to a single value pair on ingest.
+
+        Implemented in derived class.
+
+        Parameters
+        ----------
+        value
+            The value to be added
+
+        Returns
+        -------
+        Any
+        """
+        pass
+
+    @abstractmethod
+    def getter(self) -> Any:
+        """
+        Logic to apply to all ingested values when computing the `bud`.
+
+        Implemented in derived class.
+
+        Returns
+        -------
+        The value of the bud
+        """
+        pass
+
+
+class SetStem(ABC):
+    """
+    Base class for collecting and applying logic to values in a `set` with a `Stem`-like interface.
+
+    Unlike the full `Stem`, this class does NOT retain information about the keys and thus does no grouping of keys based
+    on values. The direct consequence of this is that the `.petals` property is undefined and will raise an ``AttributeError``
+    if accessed. This also means there is no need to invert the `key_to_petal_dict` (because it doesn't exist), which,
+    in turn, means there is no need to run the `getter` for every key. The result is that the `bud` property only needs
+    one call to `getter`. Combined with the efficiency of storing values in a `set`, the calculation of a single value
+    derived from all values (i.e., `bud`) is much faster than using a full `Stem`.
+
+    .. Note::
+      The use of a `set` as the underlying storage mechanism means information regarding how many times a particular value
+      is present will be lost. It also means the return type of `setter` must be hashable. Both of these constraints can
+      be avoided by using `ListStem`, which still gets a significant speedup over `Stem` by dropping key information.
+
+
+    Parameters
+    ----------
+    stem_name
+        The name to be associated with the stem
+    """
+
+    def __init__(self, stem_name: Any):
+        self.stem_name = stem_name
+        self.value_set: set = set()
+        self._need_to_compute_bud_value: bool = True
+
+    def update(self, key: Any, value: Any) -> None:
+        """
+        Ingest a single key/value pair. Note that the ``key`` is not used.
+
+        Parameters
+        ----------
+        key
+            The key (unused)
+
+        value
+            The value
+
+        Returns
+        -------
+        None
+        """
+        result = self.setter(value)
+        if result is not SpilledDirt:
+            self.value_set.add(result)
+            self._need_to_compute_bud_value = True
+
+    @property
+    def petals(self) -> None:
+        """Raise an error because `SetStem` does not retain key information and therefore cannot group keys."""
+        raise AttributeError(
+            f"{self.__class__.__name__} subclasses SetStem and therefore does not define the `petals` property"
+        )
+
+    @property
+    def can_be_picked(self) -> bool:
+        """
+        Return True if there are any values to be picked.
+
+        A `Stem` could have no values even after dirt is added if all of the results were `SpilledDirt`.
+        """
+        return len(self.value_set) > 0
+
+    @property
+    def bud(self) -> Petal:
+        """Return the result of `getter` packaged in a `Petal` object."""
+        if self._need_to_compute_bud_value:
+            self._value_cache = self.getter()
+            self._need_to_compute_bud_value = False
+
+        return Petal((self._value_cache, "SETSTEM_NOT_USED"))
+
+    @abstractmethod
+    def setter(self, value: Any) -> Hashable:
+        """
+        Logic to apply to a single value pair on ingest.
+
+        Must return a Hashable object because the result will be stored in a `set`.
+
+        Implemented in derived class.
+
+        Parameters
+        ----------
+        value
+            The value to be added
+
+        Returns
+        -------
+        Any
+        """
+        pass
+
+    @abstractmethod
+    def getter(self) -> Any:
+        """
+        Logic to apply to all ingested values when computing the `bud`.
+
+        Implemented in derived class.
+
+        Returns
+        -------
+        The value of the bud
+        """
+        pass

dkist_processing_common/parsers/average_bud.py

@@ -30,8 +30,6 @@ class TaskAverageBud(TaskNearFloatBud):
         The function used to convert a header into an IP task type
     """
 
-    key_to_petal_dict: dict[str, float]
-
     def __init__(
         self,
         constant_name: str,

dkist_processing_common/parsers/cs_step.py

@@ -7,6 +7,7 @@ from datetime import timezone
 from typing import Type
 
 from dkist_processing_common.models.constants import BudName
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.tags import StemName
@@ -103,6 +104,7 @@ class CSStepFlower(Stem):
     def __init__(self, max_cs_step_time_sec: float):
         super().__init__(stem_name=StemName.cs_step)
         self.max_cs_step_time_sec = max_cs_step_time_sec
+        self.CS_step_set = set()
 
     def setter(self, fits_obj: L0FitsAccess) -> CSStep | Type[SpilledDirt]:
         """
@@ -119,7 +121,10 @@ class CSStepFlower(Stem):
         """
         if fits_obj.ip_task_type != "polcal":
             return SpilledDirt
-        return CSStep(fits_obj, max_cs_time_sec=self.max_cs_step_time_sec)
+
+        cs_step = CSStep(fits_obj, max_cs_time_sec=self.max_cs_step_time_sec)
+        self.CS_step_set.add(cs_step)
+        return cs_step
 
     def getter(self, key) -> str | float | int:
         """
@@ -133,11 +138,11 @@ class CSStepFlower(Stem):
         -------
         The cs step for the given key
         """
-        unique_steps = sorted(list(set(self.key_to_petal_dict.values())))
+        unique_steps = sorted(self.CS_step_set)
         return unique_steps.index(self.key_to_petal_dict[key])
 
 
-class NumCSStepBud(Stem):
+class NumCSStepBud(SetStem):
     """
     The total number of CS Steps present in a dataset.
 
@@ -168,17 +173,12 @@ class NumCSStepBud(Stem):
             return SpilledDirt
         return CSStep(fits_obj, max_cs_time_sec=self.max_cs_step_time_sec)
 
-    def getter(self, key) -> int:
+    def getter(self) -> int:
         """
         Return the number of CS Steps present.
 
-        Parameters
-        ----------
-        key
-            The input key
         Returns
         -------
         The number of cs steps associated with the key
         """
-        value_set = set(self.key_to_petal_dict.values())
-        return len(value_set)
+        return len(self.value_set)

dkist_processing_common/parsers/id_bud.py

@@ -4,13 +4,13 @@ from enum import StrEnum
 from typing import Callable
 from typing import Type
 
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
-from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 
 
-class ContributingIdsBud(Stem):
+class ContributingIdsBud(SetStem):
     """Base class for contributing ID buds."""
 
     def __init__(self, constant_name: str, metadata_key: str | StrEnum):
@@ -33,20 +33,15 @@ class ContributingIdsBud(Stem):
         """
         return getattr(fits_obj, self.metadata_key)
 
-    def getter(self, key) -> tuple[str, ...]:
+    def getter(self) -> tuple[str, ...]:
         """
         Get all ids seen for any type of frame.
 
-        Parameters
-        ----------
-        key
-            The input key
-
         Returns
         -------
         IDs from all types of frames
         """
-        return tuple(set(self.key_to_petal_dict.values()))
+        return tuple(self.value_set)
 
 
 class TaskContributingIdsBud(ContributingIdsBud):

dkist_processing_common/parsers/lookup_bud.py

@@ -6,14 +6,14 @@ from typing import Any
 from typing import Callable
 from typing import DefaultDict
 
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
-from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.tags import EXP_TIME_ROUND_DIGITS
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 
 
-class TimeLookupBud(Stem):
+class TimeLookupBud(SetStem):
     """
     Bud that reads two header keys from all files and creates a dictionary mapping a time KEY value to sets of a VALUE value.
 
@@ -64,20 +64,16 @@ class TimeLookupBud(Stem):
         self.mapping[rounded_key].add(value)
         return None
 
-    def getter(self, key):
+    def getter(self):
         """
-        Get the dictionary mapping created by the setter converted into hashable nested tuples.
+        Get the dictionary mapping created by the setter with values converted to JSON-able lists.
 
-        Parameters
-        ----------
-        key
-            The input key
         Returns
         -------
-        The mapping dictionary converted into hashable nested tuples
+        The mapping dictionary with values converted to JSON-able lists
         """
-        hashable_mapping = tuple((k, tuple(v)) for k, v in self.mapping.items())
-        return hashable_mapping
+        mapping_lists = {k: list(v) for k, v in self.mapping.items()}
+        return mapping_lists
 
 
 class TaskTimeLookupBud(TimeLookupBud):

dkist_processing_common/parsers/near_bud.py

@@ -4,15 +4,15 @@ from enum import StrEnum
 from statistics import mean
 from typing import Callable
 
+from dkist_processing_common.models.flower_pot import ListStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
-from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 
 
-class NearFloatBud(Stem):
+class NearFloatBud(ListStem):
     """
-    Pre-made flower that reads a single header key from all files and raises a ValueError if the values are not within a given tolerance.
+    Pre-made `ListStem` that reads a single header key from all files and raises a ValueError if the values are not within a given tolerance.
 
     This is intended for use with floats where the values may be slightly different, but should be the same.
 
@@ -54,26 +54,21 @@ class NearFloatBud(Stem):
         """
         return getattr(fits_obj, self.metadata_key)
 
-    def getter(self, key):
+    def getter(self):
         """
         Get the value for this key and raise an error if the data spans more than the given tolerance.
 
-        Parameters
-        ----------
-        key
-            The input key
         Returns
         -------
         The mean value associated with this input key
         """
-        value_list = list(self.key_to_petal_dict.values())
-        biggest_value = max(value_list)
-        smallest_value = min(value_list)
+        biggest_value = max(self.value_list)
+        smallest_value = min(self.value_list)
         if biggest_value - smallest_value > self.tolerance:
             raise ValueError(
                 f"{self.stem_name} values are not close enough. Max: {biggest_value}, Min: {smallest_value}, Tolerance: {self.tolerance}"
             )
-        return mean(value_list)
+        return mean(self.value_list)
 
 
 class TaskNearFloatBud(NearFloatBud):

dkist_processing_common/parsers/retarder.py

@@ -2,7 +2,9 @@
 
 from dkist_processing_common.models.constants import BudName
 from dkist_processing_common.models.fits_access import MetadataKey
+from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.models.task_name import TaskName
+from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
 
 
@@ -11,13 +13,9 @@ class RetarderNameBud(TaskUniqueBud):
     Bud for determining the name of the retarder used during a polcal Calibration Sequence (CS).
 
     This is *slightly* different than a simple `TaskUniqueBud` because we need to allow for CS steps when the retarder
-    is out of the beam (i.g., "clear"). We do this by forcing the set of header values to be `{clear, RETARDER_NAME}`,
-    where RETARDER_NAME is the value of this Bud.
+    is out of the beam (i.g., "clear"). We do this by returning `SpilledDirt` from the `setter` if the value is "clear".
     """
 
-    # For type-hinting later
-    key_to_petal_dict: dict[str, str]
-
     def __init__(self):
         super().__init__(
             constant_name=BudName.retarder_name,
@@ -25,12 +23,10 @@ class RetarderNameBud(TaskUniqueBud):
             ip_task_types=TaskName.polcal,
         )
 
-    def getter(self, key) -> str:
-        """Get the value for the retarder name and raise an Error if, ignoring "clear", that name is not unique."""
-        value_set = set(self.key_to_petal_dict.values())
-        value_set -= {"clear"}
-        if len(value_set) > 1:
-            raise ValueError(f"Multiple non-clear retarder names found. Names: {value_set}")
+    def setter(self, fits_obj: L0FitsAccess) -> type[SpilledDirt] | str:
+        """Drop the result if the retarder is out of the beam ("clear")."""
+        result = super().setter(fits_obj)
+        if result is not SpilledDirt and result.casefold() == "clear":
+            return SpilledDirt
 
-        raw_retarder_name = value_set.pop()
-        return raw_retarder_name
+        return result