dkist-processing-common 11.6.0rc1__py3-none-any.whl → 11.7.0rc1__py3-none-any.whl

dkist_processing_common/parsers/id_bud.py

@@ -1,36 +1,27 @@
 """Base classes for ID bud parsing."""
 
 from enum import StrEnum
+from typing import Callable
 from typing import Type
 
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
+from dkist_processing_common.parsers.task import passthrough_header_ip_task
 from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
 
 
-class IdBud(TaskUniqueBud):
-    """Base class for ID buds."""
-
-    def __init__(self, constant_name: str, metadata_key: str | StrEnum):
-        super().__init__(
-            constant_name=constant_name,
-            metadata_key=metadata_key,
-            ip_task_types=TaskName.observe,
-        )
-
-
 class ContributingIdsBud(Stem):
     """Base class for contributing ID buds."""
 
-    def __init__(self, stem_name: str, metadata_key: str | StrEnum):
-        super().__init__(stem_name=stem_name)
+    def __init__(self, constant_name: str, metadata_key: str | StrEnum):
+        super().__init__(stem_name=constant_name)
         if isinstance(metadata_key, StrEnum):
             metadata_key = metadata_key.name
         self.metadata_key = metadata_key
 
-    def setter(self, fits_obj: L0FitsAccess) -> str | Type[SpilledDirt]:
+    def setter(self, fits_obj: L0FitsAccess) -> str:
         """
         Set the id for any type of frame.
 
@@ -44,9 +35,9 @@ class ContributingIdsBud(Stem):
         """
         return getattr(fits_obj, self.metadata_key)
 
-    def getter(self, key) -> tuple:
+    def getter(self, key) -> tuple[str, ...]:
         """
-        Get all ids seen in non observe frames.
+        Get all ids seen for any type of frame.
 
         Parameters
         ----------
@@ -55,6 +46,33 @@ class ContributingIdsBud(Stem):
 
         Returns
         -------
-        IDs from non observe frames
+        IDs from all types of frames
         """
         return tuple(set(self.key_to_petal_dict.values()))
+
+
+class TaskContributingIdsBud(ContributingIdsBud):
+    """Base class for contributing ID buds for a particular task type."""
+
+    def __init__(
+        self,
+        constant_name: str,
+        metadata_key: str | StrEnum,
+        ip_task_types: str | list[str],
+        task_type_parsing_function: Callable = passthrough_header_ip_task,
+    ):
+        super().__init__(constant_name=constant_name, metadata_key=metadata_key)
+
+        if isinstance(ip_task_types, str):
+            ip_task_types = [ip_task_types]
+        self.ip_task_types = [task.casefold() for task in ip_task_types]
+        self.parsing_function = task_type_parsing_function
+
+    def setter(self, fits_obj: L0FitsAccess) -> str | Type[SpilledDirt]:
+        """Ingest an object only if its parsed IP task type matches what's desired."""
+        task = self.parsing_function(fits_obj)
+
+        if task.casefold() in self.ip_task_types:
+            return super().setter(fits_obj)
+
+        return SpilledDirt

dkist_processing_common/parsers/l0_fits_access.py

@@ -27,6 +27,6 @@ class L0FitsAccess(L1FitsAccess):
         auto_squeeze: bool = True,
     ):
         super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
-        self._set_metadata_key_value(MetadataKey.ip_task_type)
-        self._set_metadata_key_value(MetadataKey.ip_start_time)
-        self._set_metadata_key_value(MetadataKey.ip_end_time)
+        self.ip_task_type: str = self.header[MetadataKey.ip_task_type]
+        self.ip_start_time: str = self.header[MetadataKey.ip_start_time]
+        self.ip_end_time: str = self.header[MetadataKey.ip_end_time]

dkist_processing_common/parsers/l1_fits_access.py

@@ -2,9 +2,12 @@
 
 from astropy.io import fits
 
+from dkist_processing_common.models.fits_access import HEADER_KEY_NOT_FOUND
 from dkist_processing_common.models.fits_access import FitsAccessBase
 from dkist_processing_common.models.fits_access import MetadataKey
 
+NOT_A_FLOAT = -999
+
 
 class L1FitsAccess(FitsAccessBase):
     """
@@ -28,25 +31,48 @@ class L1FitsAccess(FitsAccessBase):
     ):
         super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
 
-        self._set_metadata_key_value(MetadataKey.elevation)
-        self._set_metadata_key_value(MetadataKey.azimuth)
-        self._set_metadata_key_value(MetadataKey.table_angle)
-        self._set_metadata_key_value(MetadataKey.gos_level3_status)
-        self._set_metadata_key_value(MetadataKey.gos_level3_lamp_status)
-        self._set_metadata_key_value(MetadataKey.gos_polarizer_status)
-        self._set_metadata_key_value(MetadataKey.gos_retarder_status)
-        self._set_metadata_key_value(MetadataKey.gos_level0_status)
-        self._set_metadata_key_value(MetadataKey.time_obs)
-        self._set_metadata_key_value(MetadataKey.ip_id)
-        self._set_metadata_key_value(MetadataKey.instrument)
-        self._set_metadata_key_value(MetadataKey.wavelength)
-        self._set_metadata_key_value(MetadataKey.proposal_id)
-        self._set_metadata_key_value(MetadataKey.experiment_id)
-        self._set_metadata_key_value(MetadataKey.num_dsps_repeats)
-        self._set_metadata_key_value(MetadataKey.current_dsps_repeat)
-        self._set_metadata_key_value(MetadataKey.fpa_exposure_time_ms)
-        self._set_metadata_key_value(MetadataKey.sensor_readout_exposure_time_ms)
-        self._set_metadata_key_value(MetadataKey.num_raw_frames_per_fpa)
+        self.elevation: float = self.header[MetadataKey.elevation]
+        self.azimuth: float = self.header[MetadataKey.azimuth]
+        self.table_angle: float = self.header[MetadataKey.table_angle]
+        self.gos_level3_status: str = self.header[MetadataKey.gos_level3_status]
+        self.gos_level3_lamp_status: str = self.header[MetadataKey.gos_level3_lamp_status]
+        self.gos_polarizer_status: str = self.header[MetadataKey.gos_polarizer_status]
+        self.gos_retarder_status: str = self.header[MetadataKey.gos_retarder_status]
+        self.gos_level0_status: str = self.header[MetadataKey.gos_level0_status]
+        self.time_obs: str = self.header[MetadataKey.time_obs]
+        self.ip_id: str = self.header[MetadataKey.ip_id]
+        self.instrument: str = self.header[MetadataKey.instrument]
+        self.wavelength: float = self.header[MetadataKey.wavelength]
+        self.proposal_id: str = self.header[MetadataKey.proposal_id]
+        self.experiment_id: str = self.header[MetadataKey.experiment_id]
+        self.num_dsps_repeats: int = self.header[MetadataKey.num_dsps_repeats]
+        self.current_dsps_repeat: int = self.header[MetadataKey.current_dsps_repeat]
+        self.fpa_exposure_time_ms: float = self.header[MetadataKey.fpa_exposure_time_ms]
+        self.sensor_readout_exposure_time_ms: float = self.header[
+            MetadataKey.sensor_readout_exposure_time_ms
+        ]
+        self.num_raw_frames_per_fpa: int = self.header[MetadataKey.num_raw_frames_per_fpa]
+        self.camera_id: str = self.header[MetadataKey.camera_id]
+        self.camera_name: str = self.header[MetadataKey.camera_name]
+        self.camera_bit_depth: int = self.header[MetadataKey.camera_bit_depth]
+        self.hardware_binning_x: int = self.header[MetadataKey.hardware_binning_x]
+        self.hardware_binning_y: int = self.header[MetadataKey.hardware_binning_y]
+        self.software_binning_x: int = self.header[MetadataKey.software_binning_x]
+        self.software_binning_y: int = self.header[MetadataKey.software_binning_y]
+        self.observing_program_execution_id: str = self.header[
+            MetadataKey.observing_program_execution_id
+        ]
+        self.telescope_tracking_mode: str = self.header.get(
+            MetadataKey.telescope_tracking_mode, HEADER_KEY_NOT_FOUND
+        )
+        self.coude_table_tracking_mode: str = self.header.get(
+            MetadataKey.coude_table_tracking_mode, HEADER_KEY_NOT_FOUND
+        )
+        self.telescope_scanning_mode: str = self.header.get(
+            MetadataKey.telescope_scanning_mode, HEADER_KEY_NOT_FOUND
+        )
+        self.light_level: float = self.header[MetadataKey.light_level]
+        self.hls_version: str = self.header[MetadataKey.hls_version]
 
     @property
     def gos_polarizer_angle(self) -> float:
@@ -54,7 +80,7 @@ class L1FitsAccess(FitsAccessBase):
         try:
             return float(self.header[MetadataKey.gos_polarizer_angle])
         except ValueError:
-            return -999  # The angle is only used if the polarizer is in the beam
+            return NOT_A_FLOAT  # The angle is only used if the polarizer is in the beam
 
     @property
     def gos_retarder_angle(self) -> float:
@@ -62,4 +88,4 @@ class L1FitsAccess(FitsAccessBase):
         try:
             return float(self.header[MetadataKey.gos_retarder_angle])
         except ValueError:
-            return -999  # The angle is only used if the retarder is in the beam
+            return NOT_A_FLOAT  # The angle is only used if the retarder is in the beam

dkist_processing_common/parsers/near_bud.py

@@ -66,14 +66,14 @@ class NearFloatBud(Stem):
         -------
         The mean value associated with this input key
         """
-        value_set = list(self.key_to_petal_dict.values())
-        biggest_value = max(value_set)
-        smallest_value = min(value_set)
+        value_list = list(self.key_to_petal_dict.values())
+        biggest_value = max(value_list)
+        smallest_value = min(value_list)
         if biggest_value - smallest_value > self.tolerance:
             raise ValueError(
                 f"{self.stem_name} values are not close enough. Max: {biggest_value}, Min: {smallest_value}, Tolerance: {self.tolerance}"
             )
-        return mean(value_set)
+        return mean(value_list)
 
 
 class TaskNearFloatBud(NearFloatBud):

dkist_processing_common/parsers/proposal_id_bud.py

@@ -2,21 +2,27 @@
 
 from dkist_processing_common.models.constants import BudName
 from dkist_processing_common.models.fits_access import MetadataKey
+from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_common.parsers.id_bud import ContributingIdsBud
-from dkist_processing_common.parsers.id_bud import IdBud
+from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
 
 
-class ProposalIdBud(IdBud):
+class ProposalIdBud(TaskUniqueBud):
     """Class to create a Bud for the proposal_id."""
 
     def __init__(self):
-        super().__init__(constant_name=BudName.proposal_id, metadata_key=MetadataKey.proposal_id)
+        super().__init__(
+            constant_name=BudName.proposal_id,
+            metadata_key=MetadataKey.proposal_id,
+            ip_task_types=TaskName.observe,
+        )
 
 
 class ContributingProposalIdsBud(ContributingIdsBud):
-    """Class to create a Bud for the proposal_ids."""
+    """Class to create a Bud for the supporting proposal_ids."""
 
     def __init__(self):
         super().__init__(
-            stem_name=BudName.contributing_proposal_ids, metadata_key=MetadataKey.proposal_id
+            constant_name=BudName.contributing_proposal_ids,
+            metadata_key=MetadataKey.proposal_id,
         )

dkist_processing_common/parsers/single_value_single_key_flower.py

@@ -2,7 +2,6 @@
 
 from enum import StrEnum
 
-from dkist_processing_common.models.fits_access import MetadataKey
 from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 

dkist_processing_common/parsers/time.py

@@ -22,7 +22,6 @@ from dkist_processing_common.parsers.single_value_single_key_flower import (
 )
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
-from dkist_processing_common.parsers.unique_bud import UniqueBud
 
 
 class ObsIpStartTimeBud(TaskUniqueBud):
@@ -36,15 +35,52 @@ class ObsIpStartTimeBud(TaskUniqueBud):
         )
 
 
-class CadenceBudBase(UniqueBud):
-    """Base class for all Cadence Buds."""
+class TaskDatetimeBudBase(Stem):
+    """
+    Base class for making datetime-related buds.
 
-    def __init__(self, constant_name: str):
-        super().__init__(constant_name, metadata_key=MetadataKey.time_obs)
+    Returns a tuple of sorted values converted from datetimes to unix seconds.
+
+    Complicated parsing of the header into a task type can be achieved by passing in a different
+    header task parsing function.
+
+    Parameters
+    ----------
+    stem_name
+        The name for the constant to be defined
+
+    metadata_key
+        The metadata key associated with the constant
+
+    ip_task_types
+        Only consider objects whose parsed header IP task type matches a string in this list
+
+    header_type_parsing_func
+        The function used to convert a header into an IP task type
+    """
+
+    key_to_petal_dict: dict[str, float]
+
+    def __init__(
+        self,
+        stem_name: str,
+        metadata_key: str | StrEnum,
+        ip_task_types: str | list[str],
+        header_task_parsing_func: Callable = passthrough_header_ip_task,
+    ):
+        super().__init__(stem_name=stem_name)
+
+        if isinstance(metadata_key, StrEnum):
+            metadata_key = metadata_key.name
+        self.metadata_key = metadata_key
+        if isinstance(ip_task_types, str):
+            ip_task_types = [ip_task_types]
+        self.ip_task_types = [task.casefold() for task in ip_task_types]
+        self.header_parsing_function = header_task_parsing_func
 
     def setter(self, fits_obj: L0FitsAccess) -> float | Type[SpilledDirt]:
         """
-        If the file is an observe file, its DATE-OBS value is stored as unix seconds.
+        Store the metadata key datetime value as unix seconds if the task type is in the desired types.
 
         Parameters
         ----------
@@ -52,16 +88,45 @@ class CadenceBudBase(UniqueBud):
             The input fits object
         Returns
         -------
-        The observe time in seconds
+        The datetime in seconds
         """
-        if fits_obj.ip_task_type.casefold() == TaskName.observe.value.casefold():
+        task = self.header_parsing_function(fits_obj)
+
+        if task.casefold() in self.ip_task_types:
             return (
                 datetime.fromisoformat(getattr(fits_obj, self.metadata_key))
                 .replace(tzinfo=timezone.utc)
                 .timestamp()
             )
+
         return SpilledDirt
 
+    def getter(self, key: Hashable) -> tuple[float, ...]:
+        """
+        Return a tuple of sorted times in unix seconds.
+
+        Parameters
+        ----------
+        key
+            The input key
+
+        Returns
+        -------
+        A tuple that is sorted times in unix seconds
+        """
+        return tuple(sorted(list(self.key_to_petal_dict.values())))
+
+
+class CadenceBudBase(TaskDatetimeBudBase):
+    """Base class for all Cadence Buds."""
+
+    def __init__(self, constant_name: str):
+        super().__init__(
+            stem_name=constant_name,
+            metadata_key=MetadataKey.time_obs,
+            ip_task_types=TaskName.observe,
+        )
+
 
 class AverageCadenceBud(CadenceBudBase):
     """Class for the average cadence Bud."""
@@ -82,7 +147,7 @@ class AverageCadenceBud(CadenceBudBase):
         -------
         The mean value of the cadences of the input frames
         """
-        return np.mean(np.diff(sorted(list(self.key_to_petal_dict.values()))))
+        return np.mean(np.diff(super().getter(key)))
 
 
 class MaximumCadenceBud(CadenceBudBase):
@@ -104,7 +169,7 @@ class MaximumCadenceBud(CadenceBudBase):
         -------
         The maximum cadence between frames
         """
-        return np.max(np.diff(sorted(list(self.key_to_petal_dict.values()))))
+        return np.max(np.diff(super().getter(key)))
 
 
 class MinimumCadenceBud(CadenceBudBase):
@@ -126,7 +191,7 @@ class MinimumCadenceBud(CadenceBudBase):
         -------
         The minimum cadence between frames
         """
-        return np.min(np.diff(sorted(list(self.key_to_petal_dict.values()))))
+        return np.min(np.diff(super().getter(key)))
 
 
 class VarianceCadenceBud(CadenceBudBase):
@@ -147,11 +212,38 @@ class VarianceCadenceBud(CadenceBudBase):
         -------
         Return the variance of the cadences over the input frames
         """
-        return np.var(np.diff(sorted(list(self.key_to_petal_dict.values()))))
+        return np.var(np.diff(super().getter(key)))
+
 
+class TaskDateBeginBud(TaskDatetimeBudBase):
+    """Class for the date begin task Bud."""
 
-class TimeFlowerBase(SingleValueSingleKeyFlower):
-    """Base task for SingleValueSingleKeyFlowers that need to round their values to avoid value jitter."""
+    def __init__(self, constant_name: str, ip_task_types: str | list[str]):
+        super().__init__(
+            stem_name=constant_name,
+            metadata_key=MetadataKey.time_obs,
+            ip_task_types=ip_task_types,
+        )
+
+    def getter(self, key) -> str:
+        """
+        Return the earliest date begin for the ip task type converted from unix seconds to datetime string.
+
+        Parameters
+        ----------
+        key
+            The input key
+        Returns
+        -------
+        Return the minimum date begin as a datetime string
+        """
+        min_time = super().getter(key)[0]
+        min_time_dt = datetime.fromtimestamp(min_time, tz=timezone.utc)
+        return min_time_dt.strftime("%Y-%m-%dT%H:%M:%S.%f")
+
+
+class RoundTimeFlowerBase(SingleValueSingleKeyFlower):
+    """Base flower for SingleValueSingleKeyFlowers that need to round their values to avoid value jitter."""
 
     def setter(self, fits_obj: L0FitsAccess):
         """
@@ -169,7 +261,7 @@ class TimeFlowerBase(SingleValueSingleKeyFlower):
         return round(raw_value, EXP_TIME_ROUND_DIGITS)
 
 
-class ExposureTimeFlower(TimeFlowerBase):
+class ExposureTimeFlower(RoundTimeFlowerBase):
     """For tagging the frame FPA exposure time."""
 
     def __init__(self):
@@ -178,7 +270,7 @@ class ExposureTimeFlower(TimeFlowerBase):
         )
 
 
-class ReadoutExpTimeFlower(TimeFlowerBase):
+class ReadoutExpTimeFlower(RoundTimeFlowerBase):
     """For tagging the exposure time of each readout that contributes to an FPA."""
 
     def __init__(self):
@@ -188,18 +280,18 @@ class ReadoutExpTimeFlower(TimeFlowerBase):
         )
 
 
-class TaskTimeBudBase(Stem):
+class TaskRoundTimeBudBase(Stem):
     """
-    Base class for making time-related buds that are computed for specific task types.
+    Base class for making buds that need a set of rounded times for computing for specific task types.
 
-    By "time-related" we mean values that generally need rounding when ingested into the database.
+    Metadata key values are already floats.  Returns tuple of sorted unique rounded values.
 
     Complicated parsing of the header into a task type can be achieved by passing in a different
     header task parsing function.
 
     Parameters
     ----------
-    constant_name
+    stem_name
         The name for the constant to be defined
 
     metadata_key
@@ -212,6 +304,8 @@ class TaskTimeBudBase(Stem):
         The function used to convert a header into an IP task type
     """
 
+    key_to_petal_dict: dict[str, float]
+
     def __init__(
         self,
         stem_name: str,
@@ -229,8 +323,18 @@ class TaskTimeBudBase(Stem):
         self.ip_task_types = [task.casefold() for task in ip_task_types]
         self.header_parsing_function = header_task_parsing_func
 
-    def setter(self, fits_obj: L0FitsAccess):
-        """Return the desired metadata key only if the parsed task type matches the Bud's task type."""
+    def setter(self, fits_obj: L0FitsAccess) -> float | Type[SpilledDirt]:
+        """
+        Store the metadata key value if the parsed task type is in the desired types.
+
+        Parameters
+        ----------
+        fits_obj
+            The input fits object
+        Returns
+        -------
+        The rounded time
+        """
         task = self.header_parsing_function(fits_obj)
 
         if task.casefold() in self.ip_task_types:
@@ -240,12 +344,22 @@ class TaskTimeBudBase(Stem):
         return SpilledDirt
 
     def getter(self, key: Hashable) -> tuple[float, ...]:
-        """Return a tuple of all the unique values found."""
-        value_tuple = tuple(sorted(set(self.key_to_petal_dict.values())))
-        return value_tuple
+        """
+        Return a tuple of the sorted unique values found.
+
+        Parameters
+        ----------
+        key
+            The input key
+
+        Returns
+        -------
+        A tuple that is the sorted set of unique times
+        """
+        return tuple(sorted(set(self.key_to_petal_dict.values())))
 
 
-class TaskExposureTimesBud(TaskTimeBudBase):
+class TaskExposureTimesBud(TaskRoundTimeBudBase):
     """Produce a tuple of all FPA exposure times present in the dataset for a specific ip task type."""
 
     def __init__(
@@ -262,7 +376,7 @@ class TaskExposureTimesBud(TaskTimeBudBase):
         )
 
 
-class TaskReadoutExpTimesBud(TaskTimeBudBase):
+class TaskReadoutExpTimesBud(TaskRoundTimeBudBase):
     """Produce a tuple of all sensor readout exposure times present in the dataset for a specific task type."""
 
     def __init__(

dkist_processing_common/tasks/base.py

@@ -10,14 +10,12 @@ from typing import Any
 from typing import Generator
 from typing import Iterable
 from typing import Type
-from typing import TypeAlias
 
 from dkist_processing_core import TaskBase
 from opentelemetry.metrics import CallbackOptions
 from opentelemetry.metrics import Counter
 from opentelemetry.metrics import ObservableGauge
 from opentelemetry.metrics import Observation
-from pydantic import BaseModel
 
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common._util.tags import TagDB
@@ -27,13 +25,14 @@ from dkist_processing_common.config import common_configurations
 from dkist_processing_common.models.constants import ConstantsBase
 from dkist_processing_common.models.tags import StemName
 from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.models.telemetry import ObservableProgress
 from dkist_processing_common.tasks.mixin.metadata_store import MetadataStoreMixin
 
 __all__ = ["WorkflowTaskBase", "tag_type_hint"]
 
 logger = logging.getLogger(__name__)
 
-tag_type_hint: TypeAlias = Iterable[str] | str
+tag_type_hint = Iterable[str] | str
 
 
 class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
@@ -92,6 +91,20 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
             unit="1",
             description="The number of writes executed in the processing stack.",
         )
+        self.outer_loop_progress = ObservableProgress()
+        self.outer_loop_progress_gauge: ObservableGauge = self.meter.create_observable_gauge(
+            name=self.format_metric_name("tasks.outer.loop.progress"),
+            description="The progress of a task through the main processing loop.",
+            callbacks=[lambda options: self.outer_loop_run_progress(options)],
+        )
+
+    def outer_loop_run_progress(
+        self, options: CallbackOptions
+    ) -> Generator[Observation, None, None]:
+        """Observe the progress of the current task as a percentage."""
+        yield Observation(
+            self.outer_loop_progress.percent_complete, attributes=self.base_telemetry_attributes
+        )
 
     @property
     def constants_model_class(self) -> Type[ConstantsBase]:
@@ -135,6 +148,11 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
             with self.telemetry_span("Record Provenance"):
                 self._record_provenance()
 
+    def post_run(self) -> None:
+        """Execute and post-task bookkeeping required."""
+        super().post_run()
+        self.outer_loop_progress.set_complete()
+
     def read(
         self, tags: tag_type_hint, decoder: callable = path_decoder, **decoder_kwargs
     ) -> Generator[Any, None, None]: