dkist-processing-common 12.0.0rc5__py3-none-any.whl → 12.2.0__py3-none-any.whl

dkist_processing_common/codecs/fits.py

@@ -31,15 +31,30 @@ def fits_hdulist_encoder(hdu_list: fits.HDUList) -> bytes:
     return iobase_encoder(file_obj)
 
 
-def fits_hdu_decoder(path: Path, hdu: int | None = None) -> fits.PrimaryHDU | fits.CompImageHDU:
+def fits_hdu_decoder(
+    path: Path,
+    hdu: int | None = None,
+    checksum: bool = True,
+    disable_image_compression: bool = False,
+) -> fits.PrimaryHDU | fits.CompImageHDU:
     """Read a Path with `fits` to produce an `HDUList`."""
-    hdu_list = fits.open(path, checksum=True)
+    hdu_list = fits.open(
+        path, checksum=checksum, disable_image_compression=disable_image_compression
+    )
     return _extract_hdu(hdu_list, hdu)
 
 
-def fits_array_decoder(path: Path, hdu: int | None = None, auto_squeeze: bool = True) -> np.ndarray:
+def fits_array_decoder(
+    path: Path,
+    hdu: int | None = None,
+    auto_squeeze: bool = True,
+    checksum: bool = True,
+    disable_image_compression: bool = False,
+) -> np.ndarray:
     """Read a Path with `fits` and return the `.data` property."""
-    hdu = fits_hdu_decoder(path, hdu=hdu)
+    hdu = fits_hdu_decoder(
+        path, hdu=hdu, checksum=checksum, disable_image_compression=disable_image_compression
+    )
     data = hdu.data
 
     # This conditional is explicitly to catch summit data with a dummy first axis for WCS
@@ -50,10 +65,16 @@ def fits_array_decoder(path: Path, hdu: int | None = None, auto_squeeze: bool =
 
 
 def fits_access_decoder(
-    path: Path, fits_access_class: Type[FitsAccessBase], **fits_access_kwargs
+    path: Path,
+    fits_access_class: Type[FitsAccessBase],
+    checksum: bool = True,
+    disable_image_compression: bool = False,
+    **fits_access_kwargs,
 ) -> FitsAccessBase:
     """Read a Path with `fits` and ingest into a `FitsAccessBase`-type object."""
-    hdu = fits_hdu_decoder(path)
+    hdu = fits_hdu_decoder(
+        path, checksum=checksum, disable_image_compression=disable_image_compression
+    )
     return fits_access_class(hdu=hdu, name=str(path), **fits_access_kwargs)
 
 

dkist_processing_common/models/constants.py

@@ -285,18 +285,24 @@ class ConstantsBase:
     def dark_observing_program_execution_ids(self) -> list[str]:
         """Return the observing program execution ids constant for the dark task."""
         observing_programs = self._db_dict[BudName.dark_observing_program_execution_ids]
+        if isinstance(observing_programs, str):
+            observing_programs = [observing_programs]
         return list(observing_programs)
 
     @property
     def solar_gain_observing_program_execution_ids(self) -> list[str]:
         """Return the observing program execution ids constant for the solar_gain task."""
         observing_programs = self._db_dict[BudName.solar_gain_observing_program_execution_ids]
+        if isinstance(observing_programs, str):
+            observing_programs = [observing_programs]
         return list(observing_programs)
 
     @property
     def polcal_observing_program_execution_ids(self) -> list[str]:
         """Return the observing program execution ids constant."""
         observing_programs = self._db_dict[BudName.polcal_observing_program_execution_ids]
+        if isinstance(observing_programs, str):
+            observing_programs = [observing_programs]
         return list(observing_programs)
 
     @property
@@ -333,8 +339,8 @@ class ConstantsBase:
     def dark_num_raw_frames_per_fpa(self) -> dict[float, list]:
         """Return the dictionary of exposure times to number of raw frames per fpa."""
         raw_return = self._db_dict[BudName.dark_num_raw_frames_per_fpa]
-        # convert nested lists to dictionary
-        return {k: v for k, v in raw_return}
+        # convert JSON string keys back to float
+        return {float(k): v for k, v in raw_return.items()}
 
     @property
     def solar_gain_num_raw_frames_per_fpa(self) -> int:
@@ -467,14 +473,14 @@ class ConstantsBase:
         return self._db_dict[BudName.solar_gain_gos_polarizer_status]
 
     @property
-    def dark_gos_polarizer_angle(self) -> float:
+    def dark_gos_polarizer_angle(self) -> str:
         """Return the gos polarizer angle constant for the dark task."""
-        return self._db_dict[BudName.dark_gos_polarizer_angle]
+        return str(self._db_dict[BudName.dark_gos_polarizer_angle])
 
     @property
-    def solar_gain_gos_polarizer_angle(self) -> float:
+    def solar_gain_gos_polarizer_angle(self) -> str:
         """Return the gos polarizer angle constant for the solar gain task."""
-        return self._db_dict[BudName.solar_gain_gos_polarizer_angle]
+        return str(self._db_dict[BudName.solar_gain_gos_polarizer_angle])
 
     @property
     def dark_gos_retarder_status(self) -> str:
@@ -487,14 +493,14 @@ class ConstantsBase:
         return self._db_dict[BudName.solar_gain_gos_retarder_status]
 
     @property
-    def dark_gos_retarder_angle(self) -> float:
+    def dark_gos_retarder_angle(self) -> str:
         """Return the gos retarder angle constant for the dark task."""
-        return self._db_dict[BudName.dark_gos_retarder_angle]
+        return str(self._db_dict[BudName.dark_gos_retarder_angle])
 
     @property
-    def solar_gain_gos_retarder_angle(self) -> float:
+    def solar_gain_gos_retarder_angle(self) -> str:
         """Return the gos retarder angle constant for the solar gain task."""
-        return self._db_dict[BudName.solar_gain_gos_retarder_angle]
+        return str(self._db_dict[BudName.solar_gain_gos_retarder_angle])
 
     @property
     def dark_gos_level0_status(self) -> str:

dkist_processing_common/models/extras.py

@@ -0,0 +1,35 @@
+"""Autocomplete access to dataset extra header sections."""
+
+from enum import StrEnum
+
+
+class DatasetExtraHeaderSection(StrEnum):
+    """Enum defining the possible header sections for dataset extras."""
+
+    common = "common"
+    aggregate = "aggregate"
+    iptask = "iptask"
+    gos = "gos"
+    wavecal = "wavecal"
+    atlas = "atlas"
+    test = "test"
+
+
+class DatasetExtraType(StrEnum):
+    """Enum defining options for dataset extra names."""
+
+    dark = "DARK"
+    background_light = "BACKGROUND LIGHT"
+    solar_gain = "SOLAR GAIN"
+    characteristic_spectra = "CHARACTERISTIC SPECTRA"
+    modulation_state_offsets = "MODULATION STATE OFFSETS"
+    beam_angles = "BEAM ANGLES"
+    spectral_curvature_shifts = "SPECTRAL CURVATURE SHIFTS"
+    wavelength_calibration_input_spectrum = "WAVELENGTH CALIBRATION INPUT SPECTRUM"
+    wavelength_calibration_reference_spectrum = "WAVELENGTH CALIBRATION REFERENCE SPECTRUM"
+    reference_wavelength_vector = "REFERENCE WAVELENGTH VECTOR"
+    demodulation_matrices = "DEMODULATION MATRICES"
+    polcal_as_science = "POLCAL AS SCIENCE"
+    bad_pixel_map = "BAD PIXEL MAP"
+    beam_offsets = "BEAM OFFSETS"
+    spectral_curvature_scales = "SPECTRAL CURVATURE SCALES"

dkist_processing_common/models/flower_pot.py

@@ -1,10 +1,14 @@
 """
 Framework for grouping multiple keys and values with arbitrary logic.
 
-Defines:
-    Stem -> ABC for groupings that depend on both the key and (maybe) value. Subgroups (Petals) are implied but not enforced.
+The key components are:
+    **Stem:** ABC for groupings that depend on both the key and (maybe) value. Subgroups (Petals) are implied but not enforced.
 
-    FlowerPot -> Container for Stem children (Flowers)
+    **ListStem:** ABC for groups that depend on value only. More limited, but faster than `Stem` for cases where the keys don't matter.
+
+    **SetStem:** ABC for groups that depend on value only and the values are well represented by a `set`. Even more limited, but faster than `Stem` for cases where the keys don't matter.
+
+    **FlowerPot:** Container for Stem children (Flowers)
 """
 
 from __future__ import annotations
@@ -17,7 +21,7 @@ from typing import Any
 
 
 class FlowerPot:
-    """Base class to hold multiple sets (stems) of key, value pairs."""
+    """Base class to hold multiple sets (Stems) of key, value pairs."""
 
     def __init__(self):
         self.stems: list[Stem] = list()
@@ -55,9 +59,9 @@ class FlowerPot:
 
 class SpilledDirt:
     """
-    A custom class for when a Flower wants the FlowerPot to skip that particular key/value.
+    A custom class for when a Stem wants the FlowerPot to skip that particular key/value.
 
-    Exists because None, False, [], (), etc. etc. are all valid Flower return values
+    Exists because None, False, [], (), etc. etc. are all valid Stem return values
     """
 
 
@@ -90,7 +94,7 @@ class Petal:
 
 class Stem(ABC):
     """
-    Base group for grouping keys via arbitrary logic on the total collection of keys and values.
+    Base class for grouping keys via arbitrary logic on the total collection of keys and values.
 
     Parameters
     ----------
@@ -133,6 +137,15 @@ class Stem(ABC):
 
         return self._petal_cache
 
+    @property
+    def can_be_picked(self) -> bool:
+        """
+        Return True if there are any values to be picked.
+
+        A `Stem` could have no values even after dirt is added if all of the results were `SpilledDirt`.
+        """
+        return len(self.petals) > 0
+
     def _generate_petal_list(self) -> None:
         """
         Generate a list of petals.
@@ -146,7 +159,7 @@ class Stem(ABC):
         changes, `key_to_petal_dict`, is unhashable.
         """
         petal_to_key_dict = defaultdict(list)
-        for key, petal in self.key_to_petal_dict.items():
+        for key in self.key_to_petal_dict.keys():
             petal = self.getter(key)
             petal_to_key_dict[petal].append(key)
 
@@ -180,7 +193,7 @@ class Stem(ABC):
     @abstractmethod
     def getter(self, key: Hashable) -> Hashable:
         """
-        Logic to apply to all ingested values when picking the Flower.
+        Logic to apply to all ingested values when picking the Stem.
 
         Implemented in derived class.
 
@@ -193,3 +206,211 @@ class Stem(ABC):
         The value
         """
         pass
+
+
+class ListStem(ABC):
+    """
+    Base class for collecting and applying logic to values in a `list` with a `Stem`-like interface.
+
+    Unlike the full `Stem`, this class does NOT retain information about the keys and thus does no grouping of keys based
+    on values. The direct consequence of this is that the `.petals` property is undefined and will raise an ``AttributeError``
+    if accessed. This also means there is no need to invert the `key_to_petal_dict` (because it doesn't exist), which,
+    in turn, means there is no need to run the `getter` for every key. The result is that the `bud` property only needs
+    one call to `getter`. Thus, the calculation of a single value derived from all values (i.e., `bud`) is much faster
+    than using a full `Stem`.
+
+    Parameters
+    ----------
+    stem_name
+        The name to be associated with the stem
+    """
+
+    def __init__(self, stem_name: Any):
+        self.stem_name = stem_name
+        self.value_list: list = list()
+        self._need_to_compute_bud_value: bool = True
+
+    def update(self, key: Any, value: Any) -> None:
+        """
+        Ingest a single key/value pair. Note that the ``key`` is not used.
+
+        Parameters
+        ----------
+        key
+            The key (unused)
+
+        value
+            The value
+
+        Returns
+        -------
+        None
+        """
+        result = self.setter(value)
+        if result is not SpilledDirt:
+            self.value_list.append(result)
+            self._need_to_compute_bud_value = True
+
+    @property
+    def petals(self) -> None:
+        """Raise an error because `ListStem` does not retain key information and therefore cannot group keys."""
+        raise AttributeError(
+            f"{self.__class__.__name__} subclasses ListStem and therefore does not define the `petals` property"
+        )
+
+    @property
+    def can_be_picked(self) -> bool:
+        """
+        Return True if there are any values to be picked.
+
+        A `Stem` could have no values even after dirt is added if all of the results were `SpilledDirt`.
+        """
+        return len(self.value_list) > 0
+
+    @property
+    def bud(self) -> Petal:
+        """Return the result of `getter` packaged in a `Petal` object."""
+        if self._need_to_compute_bud_value:
+            self._value_cache = self.getter()
+            self._need_to_compute_bud_value = False
+
+        return Petal((self._value_cache, "LISTSTEM_NOT_USED"))
+
+    @abstractmethod
+    def setter(self, value: Any) -> Any:
+        """
+        Logic to apply to a single value pair on ingest.
+
+        Implemented in derived class.
+
+        Parameters
+        ----------
+        value
+            The value to be added
+
+        Returns
+        -------
+        Any
+        """
+        pass
+
+    @abstractmethod
+    def getter(self) -> Any:
+        """
+        Logic to apply to all ingested values when computing the `bud`.
+
+        Implemented in derived class.
+
+        Returns
+        -------
+        The value of the bud
+        """
+        pass
+
+
+class SetStem(ABC):
+    """
+    Base class for collecting and applying logic to values in a `set` with a `Stem`-like interface.
+
+    Unlike the full `Stem`, this class does NOT retain information about the keys and thus does no grouping of keys based
+    on values. The direct consequence of this is that the `.petals` property is undefined and will raise an ``AttributeError``
+    if accessed. This also means there is no need to invert the `key_to_petal_dict` (because it doesn't exist), which,
+    in turn, means there is no need to run the `getter` for every key. The result is that the `bud` property only needs
+    one call to `getter`. Combined with the efficiency of storing values in a `set`, the calculation of a single value
+    derived from all values (i.e., `bud`) is much faster than using a full `Stem`.
+
+    .. Note::
+      The use of a `set` as the underlying storage mechanism means information regarding how many times a particular value
+      is present will be lost. It also means the return type of `setter` must be hashable. Both of these constraints can
+      be avoided by using `ListStem`, which still gets a significant speedup over `Stem` by dropping key information.
+
+
+    Parameters
+    ----------
+    stem_name
+        The name to be associated with the stem
+    """
+
+    def __init__(self, stem_name: Any):
+        self.stem_name = stem_name
+        self.value_set: set = set()
+        self._need_to_compute_bud_value: bool = True
+
+    def update(self, key: Any, value: Any) -> None:
+        """
+        Ingest a single key/value pair. Note that the ``key`` is not used.
+
+        Parameters
+        ----------
+        key
+            The key (unused)
+
+        value
+            The value
+
+        Returns
+        -------
+        None
+        """
+        result = self.setter(value)
+        if result is not SpilledDirt:
+            self.value_set.add(result)
+            self._need_to_compute_bud_value = True
+
+    @property
+    def petals(self) -> None:
+        """Raise an error because `SetStem` does not retain key information and therefore cannot group keys."""
+        raise AttributeError(
+            f"{self.__class__.__name__} subclasses SetStem and therefore does not define the `petals` property"
+        )
+
+    @property
+    def can_be_picked(self) -> bool:
+        """
+        Return True if there are any values to be picked.
+
+        A `Stem` could have no values even after dirt is added if all of the results were `SpilledDirt`.
+        """
+        return len(self.value_set) > 0
+
+    @property
+    def bud(self) -> Petal:
+        """Return the result of `getter` packaged in a `Petal` object."""
+        if self._need_to_compute_bud_value:
+            self._value_cache = self.getter()
+            self._need_to_compute_bud_value = False
+
+        return Petal((self._value_cache, "SETSTEM_NOT_USED"))
+
+    @abstractmethod
+    def setter(self, value: Any) -> Hashable:
+        """
+        Logic to apply to a single value pair on ingest.
+
+        Must return a Hashable object because the result will be stored in a `set`.
+
+        Implemented in derived class.
+
+        Parameters
+        ----------
+        value
+            The value to be added
+
+        Returns
+        -------
+        Any
+        """
+        pass
+
+    @abstractmethod
+    def getter(self) -> Any:
+        """
+        Logic to apply to all ingested values when computing the `bud`.
+
+        Implemented in derived class.
+
+        Returns
+        -------
+        The value of the bud
+        """
+        pass

dkist_processing_common/models/tags.py

@@ -38,6 +38,8 @@ class StemName(StrEnum):
     dataset_inventory = "DATASET_INVENTORY"
     asdf = "ASDF"
     quality_report = "QUALITY_REPORT"
+    # Dataset extras
+    extra = "EXTRA"
 
 
 class Tag:
@@ -450,3 +452,14 @@ class Tag:
         An asdf tag
         """
         return cls.format_tag(StemName.asdf)
+
+    @classmethod
+    def extra(cls) -> str:
+        """
+        Return a dataset extra tag.
+
+        Returns
+        -------
+        A dataset extra tag
+        """
+        return cls.format_tag(StemName.extra)

dkist_processing_common/parsers/average_bud.py

@@ -30,8 +30,6 @@ class TaskAverageBud(TaskNearFloatBud):
         The function used to convert a header into an IP task type
     """
 
-    key_to_petal_dict: dict[str, float]
-
     def __init__(
         self,
         constant_name: str,

dkist_processing_common/parsers/cs_step.py

@@ -7,6 +7,7 @@ from datetime import timezone
 from typing import Type
 
 from dkist_processing_common.models.constants import BudName
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.tags import StemName
@@ -103,6 +104,7 @@ class CSStepFlower(Stem):
     def __init__(self, max_cs_step_time_sec: float):
         super().__init__(stem_name=StemName.cs_step)
         self.max_cs_step_time_sec = max_cs_step_time_sec
+        self.CS_step_set = set()
 
     def setter(self, fits_obj: L0FitsAccess) -> CSStep | Type[SpilledDirt]:
         """
@@ -119,7 +121,10 @@ class CSStepFlower(Stem):
         """
         if fits_obj.ip_task_type != "polcal":
             return SpilledDirt
-        return CSStep(fits_obj, max_cs_time_sec=self.max_cs_step_time_sec)
+
+        cs_step = CSStep(fits_obj, max_cs_time_sec=self.max_cs_step_time_sec)
+        self.CS_step_set.add(cs_step)
+        return cs_step
 
     def getter(self, key) -> str | float | int:
         """
@@ -133,11 +138,11 @@ class CSStepFlower(Stem):
         -------
         The cs step for the given key
         """
-        unique_steps = sorted(list(set(self.key_to_petal_dict.values())))
+        unique_steps = sorted(self.CS_step_set)
         return unique_steps.index(self.key_to_petal_dict[key])
 
 
-class NumCSStepBud(Stem):
+class NumCSStepBud(SetStem):
     """
     The total number of CS Steps present in a dataset.
 
@@ -168,17 +173,12 @@ class NumCSStepBud(Stem):
             return SpilledDirt
         return CSStep(fits_obj, max_cs_time_sec=self.max_cs_step_time_sec)
 
-    def getter(self, key) -> int:
+    def getter(self) -> int:
         """
         Return the number of CS Steps present.
 
-        Parameters
-        ----------
-        key
-            The input key
         Returns
         -------
         The number of cs steps associated with the key
         """
-        value_set = set(self.key_to_petal_dict.values())
-        return len(value_set)
+        return len(self.value_set)

dkist_processing_common/parsers/id_bud.py

@@ -1,16 +1,17 @@
 """Base classes for ID bud parsing."""
 
+from collections import Counter
 from enum import StrEnum
 from typing import Callable
 from typing import Type
 
+from dkist_processing_common.models.flower_pot import ListStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
-from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 
 
-class ContributingIdsBud(Stem):
+class ContributingIdsBud(ListStem):
     """Base class for contributing ID buds."""
 
     def __init__(self, constant_name: str, metadata_key: str | StrEnum):
@@ -33,20 +34,17 @@ class ContributingIdsBud(Stem):
         """
         return getattr(fits_obj, self.metadata_key)
 
-    def getter(self, key) -> tuple[str, ...]:
+    def getter(self) -> tuple[str, ...]:
         """
-        Get all ids seen for any type of frame.
-
-        Parameters
-        ----------
-        key
-            The input key
+        Get all ids seen for any type of frame, sorted by the number of appearances of that ID.
 
         Returns
         -------
         IDs from all types of frames
         """
-        return tuple(set(self.key_to_petal_dict.values()))
+        counts = Counter(self.value_list)  # Count the number of appearances of each ID
+        sorted_ids = tuple(str(item) for item, count in counts.most_common())
+        return sorted_ids
 
 
 class TaskContributingIdsBud(ContributingIdsBud):

dkist_processing_common/parsers/lookup_bud.py

@@ -6,14 +6,14 @@ from typing import Any
 from typing import Callable
 from typing import DefaultDict
 
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
-from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.tags import EXP_TIME_ROUND_DIGITS
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 
 
-class TimeLookupBud(Stem):
+class TimeLookupBud(SetStem):
     """
     Bud that reads two header keys from all files and creates a dictionary mapping a time KEY value to sets of a VALUE value.
 
@@ -64,20 +64,16 @@ class TimeLookupBud(Stem):
         self.mapping[rounded_key].add(value)
         return None
 
-    def getter(self, key):
+    def getter(self):
         """
-        Get the dictionary mapping created by the setter converted into hashable nested tuples.
+        Get the dictionary mapping created by the setter with values converted to JSON-able lists.
 
-        Parameters
-        ----------
-        key
-            The input key
         Returns
         -------
-        The mapping dictionary converted into hashable nested tuples
+        The mapping dictionary with values converted to JSON-able lists
         """
-        hashable_mapping = tuple((k, tuple(v)) for k, v in self.mapping.items())
-        return hashable_mapping
+        mapping_lists = {k: list(v) for k, v in self.mapping.items()}
+        return mapping_lists
 
 
 class TaskTimeLookupBud(TimeLookupBud):