dkist-processing-common 12.0.0rc6__tar.gz → 12.2.0rc2__tar.gz

{dkist_processing_common-12.0.0rc6 → dkist_processing_common-12.2.0rc2}/.readthedocs.yml

@@ -2,7 +2,7 @@ version: 2
 build:
   os: ubuntu-20.04
   tools:
-    python: "3.12"
+    python: "3.13"
   apt_packages:
     - libopenjp2-7
     - graphviz

{dkist_processing_common-12.0.0rc6 → dkist_processing_common-12.2.0rc2}/CHANGELOG.rst

@@ -1,3 +1,38 @@
+v12.1.0 (2026-01-26)
+====================
+
+Features
+--------
+
+- Add `ListStem` base class for huge speedup in cases where the keys don't matter and the `getter` logic only depends on the
+  list of values computed by `setter`. This is the case for most (all?) "Buds". (`#282 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/282>`__)
+- Add `SetStem` base class that has all the benefits of `ListStem` but also gets a speedup by storing values in a `set` for
+  cases where repeated values don't need to be tracked. (`#282 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/282>`__)
+- Speed up parsing of the `*CadenceBud`, `TaskDateBeginBud`, and `[Task]NearFloatBud` by basing these buds on `ListStem`. (`#284 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/284>`__)
+- Speed up `NumCSStepBud`, `[Task]UniqueBud`, `[Task]ContributingIdsBud`, and `TaskRoundTimeBudBase` parsing by basing
+  these buds on `SetStem`. (`#285 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/285>`__)
+- Speed up `CSStepFlower` parsing by using an internal set to keep track of the unique `CSStep` objects. This removes the
+  need to compute the unique set when computing the tag for each file. (`#286 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/286>`__)
+
+
+Misc
+----
+
+- Speed up the reading of INPUT files in Parse tasks by turning off image decompression and checksum checks. (`#280 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/280>`__)
+- Update `RetarderNameBud` to drop "clear" values (i.e., the retarder is out of the beam) in the `setter` instead of the `getter`.
+  This brings it in line with standard Bud-practice. (`#285 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/285>`__)
+- Convert the TimeLookupBud to be a SetStem constant. (`#287 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/287>`__)
+
+
+v12.0.0 (2026-01-22)
+====================
+
+Misc
+----
+
+- Upgrade dkist-processing-core to 7.0.0 which includes an upgrade of Airflow to 3.1.5 and python >= 3.13. (`#278 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/278>`__)
+
+
 v11.9.3 (2026-01-07)
 ====================
 

{dkist_processing_common-12.0.0rc6 → dkist_processing_common-12.2.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 12.0.0rc6
+Version: 12.2.0rc2
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -16,10 +16,10 @@ Description-Content-Type: text/x-rst
 Requires-Dist: asdf<4.0.0,>=3.5.0
 Requires-Dist: astropy>=7.0.0
 Requires-Dist: dkist-fits-specifications<5.0,>=4.0.0
-Requires-Dist: dkist-header-validator<6.0,>=5.0.0
-Requires-Dist: dkist-processing-core==7.0.0rc8
+Requires-Dist: dkist-header-validator<6.0,>=5.3.0
+Requires-Dist: dkist-processing-core==7.0.1
 Requires-Dist: dkist-processing-pac<4.0,>=3.1
-Requires-Dist: dkist-service-configuration<5.0,>=4.1.7
+Requires-Dist: dkist-service-configuration<5.0,>=4.2.0
 Requires-Dist: dkist-spectral-lines<4.0,>=3.0.0
 Requires-Dist: solar-wavelength-calibration<3.0,>=2.0.0
 Requires-Dist: globus-sdk<5.0.0,>=4.0.0

dkist_processing_common-12.2.0rc2/changelog/272.feature.rst

@@ -0,0 +1 @@
+Add the framework for using Dataset Extras, that is, other data that can be included with the L1 FITS files generated by the regular pipeline. A new abstract class, WriteL1DatasetExtras, provides helper functionality for use in the instrument pipelines.

{dkist_processing_common-12.0.0rc6 → dkist_processing_common-12.2.0rc2}/dkist_processing_common/codecs/fits.py

@@ -31,15 +31,30 @@ def fits_hdulist_encoder(hdu_list: fits.HDUList) -> bytes:
     return iobase_encoder(file_obj)
 
 
-def fits_hdu_decoder(path: Path, hdu: int | None = None) -> fits.PrimaryHDU | fits.CompImageHDU:
+def fits_hdu_decoder(
+    path: Path,
+    hdu: int | None = None,
+    checksum: bool = True,
+    disable_image_compression: bool = False,
+) -> fits.PrimaryHDU | fits.CompImageHDU:
     """Read a Path with `fits` to produce an `HDUList`."""
-    hdu_list = fits.open(path, checksum=True)
+    hdu_list = fits.open(
+        path, checksum=checksum, disable_image_compression=disable_image_compression
+    )
     return _extract_hdu(hdu_list, hdu)
 
 
-def fits_array_decoder(path: Path, hdu: int | None = None, auto_squeeze: bool = True) -> np.ndarray:
+def fits_array_decoder(
+    path: Path,
+    hdu: int | None = None,
+    auto_squeeze: bool = True,
+    checksum: bool = True,
+    disable_image_compression: bool = False,
+) -> np.ndarray:
     """Read a Path with `fits` and return the `.data` property."""
-    hdu = fits_hdu_decoder(path, hdu=hdu)
+    hdu = fits_hdu_decoder(
+        path, hdu=hdu, checksum=checksum, disable_image_compression=disable_image_compression
+    )
     data = hdu.data
 
     # This conditional is explicitly to catch summit data with a dummy first axis for WCS
@@ -50,10 +65,16 @@ def fits_array_decoder(path: Path, hdu: int | None = None, auto_squeeze: bool =
 
 
 def fits_access_decoder(
-    path: Path, fits_access_class: Type[FitsAccessBase], **fits_access_kwargs
+    path: Path,
+    fits_access_class: Type[FitsAccessBase],
+    checksum: bool = True,
+    disable_image_compression: bool = False,
+    **fits_access_kwargs,
 ) -> FitsAccessBase:
     """Read a Path with `fits` and ingest into a `FitsAccessBase`-type object."""
-    hdu = fits_hdu_decoder(path)
+    hdu = fits_hdu_decoder(
+        path, checksum=checksum, disable_image_compression=disable_image_compression
+    )
     return fits_access_class(hdu=hdu, name=str(path), **fits_access_kwargs)
 
 

{dkist_processing_common-12.0.0rc6 → dkist_processing_common-12.2.0rc2}/dkist_processing_common/models/constants.py

@@ -285,18 +285,24 @@ class ConstantsBase:
     def dark_observing_program_execution_ids(self) -> list[str]:
         """Return the observing program execution ids constant for the dark task."""
         observing_programs = self._db_dict[BudName.dark_observing_program_execution_ids]
+        if isinstance(observing_programs, str):
+            observing_programs = [observing_programs]
         return list(observing_programs)
 
     @property
     def solar_gain_observing_program_execution_ids(self) -> list[str]:
         """Return the observing program execution ids constant for the solar_gain task."""
         observing_programs = self._db_dict[BudName.solar_gain_observing_program_execution_ids]
+        if isinstance(observing_programs, str):
+            observing_programs = [observing_programs]
         return list(observing_programs)
 
     @property
     def polcal_observing_program_execution_ids(self) -> list[str]:
         """Return the observing program execution ids constant."""
         observing_programs = self._db_dict[BudName.polcal_observing_program_execution_ids]
+        if isinstance(observing_programs, str):
+            observing_programs = [observing_programs]
         return list(observing_programs)
 
     @property
@@ -333,8 +339,8 @@ class ConstantsBase:
     def dark_num_raw_frames_per_fpa(self) -> dict[float, list]:
         """Return the dictionary of exposure times to number of raw frames per fpa."""
         raw_return = self._db_dict[BudName.dark_num_raw_frames_per_fpa]
-        # convert nested lists to dictionary
-        return {k: v for k, v in raw_return}
+        # convert JSON string keys back to float
+        return {float(k): v for k, v in raw_return.items()}
 
     @property
     def solar_gain_num_raw_frames_per_fpa(self) -> int:

dkist_processing_common-12.2.0rc2/dkist_processing_common/models/extras.py

@@ -0,0 +1,35 @@
+"""Autocomplete access to dataset extra header sections."""
+
+from enum import StrEnum
+
+
+class DatasetExtraHeaderSection(StrEnum):
+    """Enum defining the possible header sections for dataset extras."""
+
+    common = "common"
+    aggregate = "aggregate"
+    iptask = "iptask"
+    gos = "gos"
+    wavecal = "wavecal"
+    atlas = "atlas"
+    test = "test"
+
+
+class DatasetExtraType(StrEnum):
+    """Enum defining options for dataset extra names."""
+
+    dark = "DARK"
+    background_light = "BACKGROUND LIGHT"
+    solar_gain = "SOLAR GAIN"
+    characteristic_spectra = "CHARACTERISTIC SPECTRA"
+    modulation_state_offsets = "MODULATION STATE OFFSETS"
+    beam_angles = "BEAM ANGLES"
+    spectral_curvature_shifts = "SPECTRAL CURVATURE SHIFTS"
+    wavelength_calibration_input_spectrum = "WAVELENGTH CALIBRATION INPUT SPECTRUM"
+    wavelength_calibration_reference_spectrum = "WAVELENGTH CALIBRATION REFERENCE SPECTRUM"
+    reference_wavelength_vector = "REFERENCE WAVELENGTH VECTOR"
+    demodulation_matrices = "DEMODULATION MATRICES"
+    polcal_as_science = "POLCAL AS SCIENCE"
+    bad_pixel_map = "BAD PIXEL MAP"
+    beam_offsets = "BEAM OFFSETS"
+    spectral_curvature_scales = "SPECTRAL CURVATURE SCALES"

dkist_processing_common-12.2.0rc2/dkist_processing_common/models/flower_pot.py

@@ -0,0 +1,416 @@
+"""
+Framework for grouping multiple keys and values with arbitrary logic.
+
+The key components are:
+    **Stem:** ABC for groupings that depend on both the key and (maybe) value. Subgroups (Petals) are implied but not enforced.
+
+    **ListStem:** ABC for groups that depend on value only. More limited, but faster than `Stem` for cases where the keys don't matter.
+
+    **SetStem:** ABC for groups that depend on value only and the values are well represented by a `set`. Even more limited, but faster than `Stem` for cases where the keys don't matter.
+
+    **FlowerPot:** Container for Stem children (Flowers)
+"""
+
+from __future__ import annotations
+
+from abc import ABC
+from abc import abstractmethod
+from collections import defaultdict
+from collections.abc import Hashable
+from typing import Any
+
+
+class FlowerPot:
+    """Base class to hold multiple sets (Stems) of key, value pairs."""
+
+    def __init__(self):
+        self.stems: list[Stem] = list()
+
+    def __iter__(self):
+        return self.stems.__iter__()
+
+    def __len__(self):
+        return self.stems.__len__()
+
+    def __getitem__(self, item):
+        return self.stems.__getitem__(item)
+
+    def add_dirt(self, key: Hashable, value: Any) -> None:
+        """
+        Send key and value through all Stems.
+
+        Parameters
+        ----------
+        key
+            The key
+        value
+            The value
+
+        Returns
+        -------
+        None
+        """
+        if not isinstance(key, Hashable):
+            raise TypeError(f"Type of key ({type(key)}) is not hashable")
+
+        for stem in self.stems:
+            stem.update(key, value)
+
+
+class SpilledDirt:
+    """
+    A custom class for when a Stem wants the FlowerPot to skip that particular key/value.
+
+    Exists because None, False, [], (), etc. etc. are all valid Stem return values
+    """
+
+
+class Thorn:
+    """
+    Custom class to indicate that a Bud's value should not be used.
+
+    I.e., don't pick this Bud because it's thorny. This exists for "Picky" Buds that merely perform a check and Error
+    on failure. If the check passes we don't need to do anything with the Bud's value and that value should be a Thorn.
+    """
+
+
+class Petal:
+    """
+    Base class to hold a single key, value pair.
+
+    Parameters
+    ----------
+    item
+        The key, value pair to be added
+    """
+
+    def __init__(self, item: tuple):
+        self.value = item[0]
+        self.keys = item[1]
+
+    def __repr__(self):
+        return f"Petal: {{{self.value}: {self.keys}}}"
+
+
+class Stem(ABC):
+    """
+    Base class for grouping keys via arbitrary logic on the total collection of keys and values.
+
+    Parameters
+    ----------
+    stem_name
+        The name to be associated with the stem
+    """
+
+    def __init__(self, stem_name: Any):
+        self.stem_name = stem_name
+        self.key_to_petal_dict: dict[Hashable, Hashable] = dict()
+
+        self._petal_cache: list[Petal] = []
+        self._need_to_generate_petals: bool = True
+
+    def update(self, key: Hashable, value: Any) -> None:
+        """
+        Ingest a single key/value pair.
+
+        Parameters
+        ----------
+        key
+            The key
+        value
+            The value
+
+        Returns
+        -------
+        None
+        """
+        result = self.setter(value)
+        if result is not SpilledDirt:
+            self.key_to_petal_dict[key] = result
+            self._need_to_generate_petals = True
+
+    @property
+    def petals(self) -> list[Petal]:
+        """Return subgroups and associated keys."""
+        if self._need_to_generate_petals:
+            self._generate_petal_list()
+
+        return self._petal_cache
+
+    @property
+    def can_be_picked(self) -> bool:
+        """
+        Return True if there are any values to be picked.
+
+        A `Stem` could have no values even after dirt is added if all of the results were `SpilledDirt`.
+        """
+        return len(self.petals) > 0
+
+    def _generate_petal_list(self) -> None:
+        """
+        Generate a list of petals.
+
+        Generating a petal list can be expensive because it involves inverting the `key_to_petal_dict`. To avoid doing
+        this every the `petals` property is accessed, this method produces a cached list of petals that is only
+        re-generated when the `key_to_petal_dict` has been updated.
+
+        Note: We can't make `petals` a `@cached_property` because that would make `petals` always tied to whatever the
+        state of this `Stem` was when `petals` was first accessed. And we can't use `lru_cache` because the object that
+        changes, `key_to_petal_dict`, is unhashable.
+        """
+        petal_to_key_dict = defaultdict(list)
+        for key in self.key_to_petal_dict.keys():
+            petal = self.getter(key)
+            petal_to_key_dict[petal].append(key)
+
+        self._petal_cache = [Petal(item) for item in petal_to_key_dict.items()]
+        self._need_to_generate_petals = False
+
+    @property
+    def bud(self) -> Petal:
+        """Just the first petal."""
+        bud = self.petals[0]
+        return bud
+
+    @abstractmethod
+    def setter(self, value: Any) -> Any:
+        """
+        Logic to apply to a single key/value pair on ingest.
+
+        Implemented in derived class.
+
+        Parameters
+        ----------
+        value
+            The value to be added
+
+        Returns
+        -------
+        Any
+        """
+        pass
+
+    @abstractmethod
+    def getter(self, key: Hashable) -> Hashable:
+        """
+        Logic to apply to all ingested values when picking the Stem.
+
+        Implemented in derived class.
+
+        Parameters
+        ----------
+        key
+            The key to return the value of
+        Returns
+        -------
+        The value
+        """
+        pass
+
+
+class ListStem(ABC):
+    """
+    Base class for collecting and applying logic to values in a `list` with a `Stem`-like interface.
+
+    Unlike the full `Stem`, this class does NOT retain information about the keys and thus does no grouping of keys based
+    on values. The direct consequence of this is that the `.petals` property is undefined and will raise an ``AttributeError``
+    if accessed. This also means there is no need to invert the `key_to_petal_dict` (because it doesn't exist), which,
+    in turn, means there is no need to run the `getter` for every key. The result is that the `bud` property only needs
+    one call to `getter`. Thus, the calculation of a single value derived from all values (i.e., `bud`) is much faster
+    than using a full `Stem`.
+
+    Parameters
+    ----------
+    stem_name
+        The name to be associated with the stem
+    """
+
+    def __init__(self, stem_name: Any):
+        self.stem_name = stem_name
+        self.value_list: list = list()
+        self._need_to_compute_bud_value: bool = True
+
+    def update(self, key: Any, value: Any) -> None:
+        """
+        Ingest a single key/value pair. Note that the ``key`` is not used.
+
+        Parameters
+        ----------
+        key
+            The key (unused)
+
+        value
+            The value
+
+        Returns
+        -------
+        None
+        """
+        result = self.setter(value)
+        if result is not SpilledDirt:
+            self.value_list.append(result)
+            self._need_to_compute_bud_value = True
+
+    @property
+    def petals(self) -> None:
+        """Raise an error because `ListStem` does not retain key information and therefore cannot group keys."""
+        raise AttributeError(
+            f"{self.__class__.__name__} subclasses ListStem and therefore does not define the `petals` property"
+        )
+
+    @property
+    def can_be_picked(self) -> bool:
+        """
+        Return True if there are any values to be picked.
+
+        A `Stem` could have no values even after dirt is added if all of the results were `SpilledDirt`.
+        """
+        return len(self.value_list) > 0
+
+    @property
+    def bud(self) -> Petal:
+        """Return the result of `getter` packaged in a `Petal` object."""
+        if self._need_to_compute_bud_value:
+            self._value_cache = self.getter()
+            self._need_to_compute_bud_value = False
+
+        return Petal((self._value_cache, "LISTSTEM_NOT_USED"))
+
+    @abstractmethod
+    def setter(self, value: Any) -> Any:
+        """
+        Logic to apply to a single value pair on ingest.
+
+        Implemented in derived class.
+
+        Parameters
+        ----------
+        value
+            The value to be added
+
+        Returns
+        -------
+        Any
+        """
+        pass
+
+    @abstractmethod
+    def getter(self) -> Any:
+        """
+        Logic to apply to all ingested values when computing the `bud`.
+
+        Implemented in derived class.
+
+        Returns
+        -------
+        The value of the bud
+        """
+        pass
+
+
+class SetStem(ABC):
+    """
+    Base class for collecting and applying logic to values in a `set` with a `Stem`-like interface.
+
+    Unlike the full `Stem`, this class does NOT retain information about the keys and thus does no grouping of keys based
+    on values. The direct consequence of this is that the `.petals` property is undefined and will raise an ``AttributeError``
+    if accessed. This also means there is no need to invert the `key_to_petal_dict` (because it doesn't exist), which,
+    in turn, means there is no need to run the `getter` for every key. The result is that the `bud` property only needs
+    one call to `getter`. Combined with the efficiency of storing values in a `set`, the calculation of a single value
+    derived from all values (i.e., `bud`) is much faster than using a full `Stem`.
+
+    .. Note::
+      The use of a `set` as the underlying storage mechanism means information regarding how many times a particular value
+      is present will be lost. It also means the return type of `setter` must be hashable. Both of these constraints can
+      be avoided by using `ListStem`, which still gets a significant speedup over `Stem` by dropping key information.
+
+
+    Parameters
+    ----------
+    stem_name
+        The name to be associated with the stem
+    """
+
+    def __init__(self, stem_name: Any):
+        self.stem_name = stem_name
+        self.value_set: set = set()
+        self._need_to_compute_bud_value: bool = True
+
+    def update(self, key: Any, value: Any) -> None:
+        """
+        Ingest a single key/value pair. Note that the ``key`` is not used.
+
+        Parameters
+        ----------
+        key
+            The key (unused)
+
+        value
+            The value
+
+        Returns
+        -------
+        None
+        """
+        result = self.setter(value)
+        if result is not SpilledDirt:
+            self.value_set.add(result)
+            self._need_to_compute_bud_value = True
+
+    @property
+    def petals(self) -> None:
+        """Raise an error because `SetStem` does not retain key information and therefore cannot group keys."""
+        raise AttributeError(
+            f"{self.__class__.__name__} subclasses SetStem and therefore does not define the `petals` property"
+        )
+
+    @property
+    def can_be_picked(self) -> bool:
+        """
+        Return True if there are any values to be picked.
+
+        A `Stem` could have no values even after dirt is added if all of the results were `SpilledDirt`.
+        """
+        return len(self.value_set) > 0
+
+    @property
+    def bud(self) -> Petal:
+        """Return the result of `getter` packaged in a `Petal` object."""
+        if self._need_to_compute_bud_value:
+            self._value_cache = self.getter()
+            self._need_to_compute_bud_value = False
+
+        return Petal((self._value_cache, "SETSTEM_NOT_USED"))
+
+    @abstractmethod
+    def setter(self, value: Any) -> Hashable:
+        """
+        Logic to apply to a single value pair on ingest.
+
+        Must return a Hashable object because the result will be stored in a `set`.
+
+        Implemented in derived class.
+
+        Parameters
+        ----------
+        value
+            The value to be added
+
+        Returns
+        -------
+        Any
+        """
+        pass
+
+    @abstractmethod
+    def getter(self) -> Any:
+        """
+        Logic to apply to all ingested values when computing the `bud`.
+
+        Implemented in derived class.
+
+        Returns
+        -------
+        The value of the bud
+        """
+        pass

{dkist_processing_common-12.0.0rc6 → dkist_processing_common-12.2.0rc2}/dkist_processing_common/models/tags.py

@@ -38,6 +38,8 @@ class StemName(StrEnum):
     dataset_inventory = "DATASET_INVENTORY"
     asdf = "ASDF"
     quality_report = "QUALITY_REPORT"
+    # Dataset extras
+    extra = "EXTRA"
 
 
 class Tag:
@@ -450,3 +452,14 @@ class Tag:
         An asdf tag
         """
         return cls.format_tag(StemName.asdf)
+
+    @classmethod
+    def extra(cls) -> str:
+        """
+        Return a dataset extra tag.
+
+        Returns
+        -------
+        A dataset extra tag
+        """
+        return cls.format_tag(StemName.extra)

{dkist_processing_common-12.0.0rc6 → dkist_processing_common-12.2.0rc2}/dkist_processing_common/parsers/average_bud.py

@@ -30,8 +30,6 @@ class TaskAverageBud(TaskNearFloatBud):
         The function used to convert a header into an IP task type
     """
 
-    key_to_petal_dict: dict[str, float]
-
     def __init__(
         self,
         constant_name: str,