dkist-processing-common 11.7.1rc1__py3-none-any.whl → 11.8.0rc1__py3-none-any.whl

dkist_processing_common/parsers/l0_fits_access.py

@@ -27,6 +27,6 @@ class L0FitsAccess(L1FitsAccess):
         auto_squeeze: bool = True,
     ):
         super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
-        self._set_metadata_key_value(MetadataKey.ip_task_type)
-        self._set_metadata_key_value(MetadataKey.ip_start_time)
-        self._set_metadata_key_value(MetadataKey.ip_end_time)
+        self.ip_task_type: str = self.header[MetadataKey.ip_task_type]
+        self.ip_start_time: str = self.header[MetadataKey.ip_start_time]
+        self.ip_end_time: str = self.header[MetadataKey.ip_end_time]

dkist_processing_common/parsers/l1_fits_access.py

@@ -2,9 +2,12 @@
 
 from astropy.io import fits
 
+from dkist_processing_common.models.fits_access import HEADER_KEY_NOT_FOUND
 from dkist_processing_common.models.fits_access import FitsAccessBase
 from dkist_processing_common.models.fits_access import MetadataKey
 
+NOT_A_FLOAT = -999
+
 
 class L1FitsAccess(FitsAccessBase):
     """
@@ -28,25 +31,48 @@ class L1FitsAccess(FitsAccessBase):
     ):
         super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
 
-        self._set_metadata_key_value(MetadataKey.elevation)
-        self._set_metadata_key_value(MetadataKey.azimuth)
-        self._set_metadata_key_value(MetadataKey.table_angle)
-        self._set_metadata_key_value(MetadataKey.gos_level3_status)
-        self._set_metadata_key_value(MetadataKey.gos_level3_lamp_status)
-        self._set_metadata_key_value(MetadataKey.gos_polarizer_status)
-        self._set_metadata_key_value(MetadataKey.gos_retarder_status)
-        self._set_metadata_key_value(MetadataKey.gos_level0_status)
-        self._set_metadata_key_value(MetadataKey.time_obs)
-        self._set_metadata_key_value(MetadataKey.ip_id)
-        self._set_metadata_key_value(MetadataKey.instrument)
-        self._set_metadata_key_value(MetadataKey.wavelength)
-        self._set_metadata_key_value(MetadataKey.proposal_id)
-        self._set_metadata_key_value(MetadataKey.experiment_id)
-        self._set_metadata_key_value(MetadataKey.num_dsps_repeats)
-        self._set_metadata_key_value(MetadataKey.current_dsps_repeat)
-        self._set_metadata_key_value(MetadataKey.fpa_exposure_time_ms)
-        self._set_metadata_key_value(MetadataKey.sensor_readout_exposure_time_ms)
-        self._set_metadata_key_value(MetadataKey.num_raw_frames_per_fpa)
+        self.elevation: float = self.header[MetadataKey.elevation]
+        self.azimuth: float = self.header[MetadataKey.azimuth]
+        self.table_angle: float = self.header[MetadataKey.table_angle]
+        self.gos_level3_status: str = self.header[MetadataKey.gos_level3_status]
+        self.gos_level3_lamp_status: str = self.header[MetadataKey.gos_level3_lamp_status]
+        self.gos_polarizer_status: str = self.header[MetadataKey.gos_polarizer_status]
+        self.gos_retarder_status: str = self.header[MetadataKey.gos_retarder_status]
+        self.gos_level0_status: str = self.header[MetadataKey.gos_level0_status]
+        self.time_obs: str = self.header[MetadataKey.time_obs]
+        self.ip_id: str = self.header[MetadataKey.ip_id]
+        self.instrument: str = self.header[MetadataKey.instrument]
+        self.wavelength: float = self.header[MetadataKey.wavelength]
+        self.proposal_id: str = self.header[MetadataKey.proposal_id]
+        self.experiment_id: str = self.header[MetadataKey.experiment_id]
+        self.num_dsps_repeats: int = self.header[MetadataKey.num_dsps_repeats]
+        self.current_dsps_repeat: int = self.header[MetadataKey.current_dsps_repeat]
+        self.fpa_exposure_time_ms: float = self.header[MetadataKey.fpa_exposure_time_ms]
+        self.sensor_readout_exposure_time_ms: float = self.header[
+            MetadataKey.sensor_readout_exposure_time_ms
+        ]
+        self.num_raw_frames_per_fpa: int = self.header[MetadataKey.num_raw_frames_per_fpa]
+        self.camera_id: str = self.header[MetadataKey.camera_id]
+        self.camera_name: str = self.header[MetadataKey.camera_name]
+        self.camera_bit_depth: int = self.header[MetadataKey.camera_bit_depth]
+        self.hardware_binning_x: int = self.header[MetadataKey.hardware_binning_x]
+        self.hardware_binning_y: int = self.header[MetadataKey.hardware_binning_y]
+        self.software_binning_x: int = self.header[MetadataKey.software_binning_x]
+        self.software_binning_y: int = self.header[MetadataKey.software_binning_y]
+        self.observing_program_execution_id: str = self.header[
+            MetadataKey.observing_program_execution_id
+        ]
+        self.telescope_tracking_mode: str = self.header.get(
+            MetadataKey.telescope_tracking_mode, HEADER_KEY_NOT_FOUND
+        )
+        self.coude_table_tracking_mode: str = self.header.get(
+            MetadataKey.coude_table_tracking_mode, HEADER_KEY_NOT_FOUND
+        )
+        self.telescope_scanning_mode: str = self.header.get(
+            MetadataKey.telescope_scanning_mode, HEADER_KEY_NOT_FOUND
+        )
+        self.light_level: float = self.header[MetadataKey.light_level]
+        self.hls_version: str = self.header[MetadataKey.hls_version]
 
     @property
     def gos_polarizer_angle(self) -> float:
@@ -54,7 +80,7 @@ class L1FitsAccess(FitsAccessBase):
         try:
             return float(self.header[MetadataKey.gos_polarizer_angle])
         except ValueError:
-            return -999  # The angle is only used if the polarizer is in the beam
+            return NOT_A_FLOAT  # The angle is only used if the polarizer is in the beam
 
     @property
     def gos_retarder_angle(self) -> float:
@@ -62,4 +88,4 @@ class L1FitsAccess(FitsAccessBase):
         try:
             return float(self.header[MetadataKey.gos_retarder_angle])
         except ValueError:
-            return -999  # The angle is only used if the retarder is in the beam
+            return NOT_A_FLOAT  # The angle is only used if the retarder is in the beam

dkist_processing_common/parsers/near_bud.py

@@ -66,14 +66,14 @@ class NearFloatBud(Stem):
         -------
         The mean value associated with this input key
         """
-        value_set = list(self.key_to_petal_dict.values())
-        biggest_value = max(value_set)
-        smallest_value = min(value_set)
+        value_list = list(self.key_to_petal_dict.values())
+        biggest_value = max(value_list)
+        smallest_value = min(value_list)
         if biggest_value - smallest_value > self.tolerance:
             raise ValueError(
                 f"{self.stem_name} values are not close enough. Max: {biggest_value}, Min: {smallest_value}, Tolerance: {self.tolerance}"
             )
-        return mean(value_set)
+        return mean(value_list)
 
 
 class TaskNearFloatBud(NearFloatBud):

dkist_processing_common/parsers/observing_program_id_bud.py

@@ -0,0 +1,24 @@
+"""Observing Program Id parser."""
+
+from typing import Callable
+
+from dkist_processing_common.models.fits_access import MetadataKey
+from dkist_processing_common.parsers.id_bud import TaskContributingIdsBud
+from dkist_processing_common.parsers.task import passthrough_header_ip_task
+
+
+class TaskContributingObservingProgramExecutionIdsBud(TaskContributingIdsBud):
+    """Class to create a Bud for the supporting observing_program_execution_ids."""
+
+    def __init__(
+        self,
+        constant_name: str,
+        ip_task_types: str | list[str],
+        task_type_parsing_function: Callable = passthrough_header_ip_task,
+    ):
+        super().__init__(
+            constant_name=constant_name,
+            metadata_key=MetadataKey.observing_program_execution_id,
+            ip_task_types=ip_task_types,
+            task_type_parsing_function=task_type_parsing_function,
+        )

dkist_processing_common/parsers/proposal_id_bud.py

@@ -2,21 +2,27 @@
 
 from dkist_processing_common.models.constants import BudName
 from dkist_processing_common.models.fits_access import MetadataKey
+from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_common.parsers.id_bud import ContributingIdsBud
-from dkist_processing_common.parsers.id_bud import IdBud
+from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
 
 
-class ProposalIdBud(IdBud):
+class ProposalIdBud(TaskUniqueBud):
     """Class to create a Bud for the proposal_id."""
 
     def __init__(self):
-        super().__init__(constant_name=BudName.proposal_id, metadata_key=MetadataKey.proposal_id)
+        super().__init__(
+            constant_name=BudName.proposal_id,
+            metadata_key=MetadataKey.proposal_id,
+            ip_task_types=TaskName.observe,
+        )
 
 
 class ContributingProposalIdsBud(ContributingIdsBud):
-    """Class to create a Bud for the proposal_ids."""
+    """Class to create a Bud for the supporting proposal_ids."""
 
     def __init__(self):
         super().__init__(
-            stem_name=BudName.contributing_proposal_ids, metadata_key=MetadataKey.proposal_id
+            constant_name=BudName.contributing_proposal_ids,
+            metadata_key=MetadataKey.proposal_id,
         )

dkist_processing_common/parsers/single_value_single_key_flower.py

@@ -2,7 +2,6 @@
 
 from enum import StrEnum
 
-from dkist_processing_common.models.fits_access import MetadataKey
 from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 

dkist_processing_common/parsers/time.py

@@ -22,7 +22,6 @@ from dkist_processing_common.parsers.single_value_single_key_flower import (
 )
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
-from dkist_processing_common.parsers.unique_bud import UniqueBud
 
 
 class ObsIpStartTimeBud(TaskUniqueBud):
@@ -36,15 +35,52 @@ class ObsIpStartTimeBud(TaskUniqueBud):
         )
 
 
-class CadenceBudBase(UniqueBud):
-    """Base class for all Cadence Buds."""
+class TaskDatetimeBudBase(Stem):
+    """
+    Base class for making datetime-related buds.
 
-    def __init__(self, constant_name: str):
-        super().__init__(constant_name, metadata_key=MetadataKey.time_obs)
+    Returns a tuple of sorted values converted from datetimes to unix seconds.
+
+    Complicated parsing of the header into a task type can be achieved by passing in a different
+    header task parsing function.
+
+    Parameters
+    ----------
+    stem_name
+        The name for the constant to be defined
+
+    metadata_key
+        The metadata key associated with the constant
+
+    ip_task_types
+        Only consider objects whose parsed header IP task type matches a string in this list
+
+    header_type_parsing_func
+        The function used to convert a header into an IP task type
+    """
+
+    key_to_petal_dict: dict[str, float]
+
+    def __init__(
+        self,
+        stem_name: str,
+        metadata_key: str | StrEnum,
+        ip_task_types: str | list[str],
+        task_type_parsing_function: Callable = passthrough_header_ip_task,
+    ):
+        super().__init__(stem_name=stem_name)
+
+        if isinstance(metadata_key, StrEnum):
+            metadata_key = metadata_key.name
+        self.metadata_key = metadata_key
+        if isinstance(ip_task_types, str):
+            ip_task_types = [ip_task_types]
+        self.ip_task_types = [task.casefold() for task in ip_task_types]
+        self.header_parsing_function = task_type_parsing_function
 
     def setter(self, fits_obj: L0FitsAccess) -> float | Type[SpilledDirt]:
         """
-        If the file is an observe file, its DATE-OBS value is stored as unix seconds.
+        Store the metadata key datetime value as unix seconds if the task type is in the desired types.
 
         Parameters
         ----------
@@ -52,16 +88,45 @@ class CadenceBudBase(UniqueBud):
             The input fits object
         Returns
         -------
-        The observe time in seconds
+        The datetime in seconds
         """
-        if fits_obj.ip_task_type.casefold() == TaskName.observe.value.casefold():
+        task = self.header_parsing_function(fits_obj)
+
+        if task.casefold() in self.ip_task_types:
             return (
                 datetime.fromisoformat(getattr(fits_obj, self.metadata_key))
                 .replace(tzinfo=timezone.utc)
                 .timestamp()
             )
+
         return SpilledDirt
 
+    def getter(self, key: Hashable) -> tuple[float, ...]:
+        """
+        Return a tuple of sorted times in unix seconds.
+
+        Parameters
+        ----------
+        key
+            The input key
+
+        Returns
+        -------
+        A tuple that is sorted times in unix seconds
+        """
+        return tuple(sorted(list(self.key_to_petal_dict.values())))
+
+
+class CadenceBudBase(TaskDatetimeBudBase):
+    """Base class for all Cadence Buds."""
+
+    def __init__(self, constant_name: str):
+        super().__init__(
+            stem_name=constant_name,
+            metadata_key=MetadataKey.time_obs,
+            ip_task_types=TaskName.observe,
+        )
+
 
 class AverageCadenceBud(CadenceBudBase):
     """Class for the average cadence Bud."""
@@ -82,7 +147,7 @@ class AverageCadenceBud(CadenceBudBase):
         -------
         The mean value of the cadences of the input frames
         """
-        return np.mean(np.diff(sorted(list(self.key_to_petal_dict.values()))))
+        return np.mean(np.diff(super().getter(key)))
 
 
 class MaximumCadenceBud(CadenceBudBase):
@@ -104,7 +169,7 @@ class MaximumCadenceBud(CadenceBudBase):
         -------
         The maximum cadence between frames
         """
-        return np.max(np.diff(sorted(list(self.key_to_petal_dict.values()))))
+        return np.max(np.diff(super().getter(key)))
 
 
 class MinimumCadenceBud(CadenceBudBase):
@@ -126,7 +191,7 @@ class MinimumCadenceBud(CadenceBudBase):
         -------
         The minimum cadence between frames
         """
-        return np.min(np.diff(sorted(list(self.key_to_petal_dict.values()))))
+        return np.min(np.diff(super().getter(key)))
 
 
 class VarianceCadenceBud(CadenceBudBase):
@@ -147,11 +212,44 @@ class VarianceCadenceBud(CadenceBudBase):
         -------
         Return the variance of the cadences over the input frames
         """
-        return np.var(np.diff(sorted(list(self.key_to_petal_dict.values()))))
+        return np.var(np.diff(super().getter(key)))
+
+
+class TaskDateBeginBud(TaskDatetimeBudBase):
+    """Class for the date begin task Bud."""
+
+    def __init__(
+        self,
+        constant_name: str,
+        ip_task_types: str | list[str],
+        task_type_parsing_function: Callable = passthrough_header_ip_task,
+    ):
+        super().__init__(
+            stem_name=constant_name,
+            metadata_key=MetadataKey.time_obs,
+            ip_task_types=ip_task_types,
+            task_type_parsing_function=task_type_parsing_function,
+        )
+
+    def getter(self, key) -> str:
+        """
+        Return the earliest date begin for the ip task type converted from unix seconds to datetime string.
+
+        Parameters
+        ----------
+        key
+            The input key
+        Returns
+        -------
+        Return the minimum date begin as a datetime string
+        """
+        min_time = super().getter(key)[0]
+        min_time_dt = datetime.fromtimestamp(min_time, tz=timezone.utc)
+        return min_time_dt.strftime("%Y-%m-%dT%H:%M:%S.%f")
 
 
-class TimeFlowerBase(SingleValueSingleKeyFlower):
-    """Base task for SingleValueSingleKeyFlowers that need to round their values to avoid value jitter."""
+class RoundTimeFlowerBase(SingleValueSingleKeyFlower):
+    """Base flower for SingleValueSingleKeyFlowers that need to round their values to avoid value jitter."""
 
     def setter(self, fits_obj: L0FitsAccess):
         """
@@ -169,7 +267,7 @@ class TimeFlowerBase(SingleValueSingleKeyFlower):
         return round(raw_value, EXP_TIME_ROUND_DIGITS)
 
 
-class ExposureTimeFlower(TimeFlowerBase):
+class ExposureTimeFlower(RoundTimeFlowerBase):
     """For tagging the frame FPA exposure time."""
 
     def __init__(self):
@@ -178,7 +276,7 @@ class ExposureTimeFlower(TimeFlowerBase):
         )
 
 
-class ReadoutExpTimeFlower(TimeFlowerBase):
+class ReadoutExpTimeFlower(RoundTimeFlowerBase):
     """For tagging the exposure time of each readout that contributes to an FPA."""
 
     def __init__(self):
@@ -188,18 +286,18 @@ class ReadoutExpTimeFlower(TimeFlowerBase):
         )
 
 
-class TaskTimeBudBase(Stem):
+class TaskRoundTimeBudBase(Stem):
     """
-    Base class for making time-related buds that are computed for specific task types.
+    Base class for making buds that need a set of rounded times for computing for specific task types.
 
-    By "time-related" we mean values that generally need rounding when ingested into the database.
+    Metadata key values are already floats.  Returns tuple of sorted unique rounded values.
 
     Complicated parsing of the header into a task type can be achieved by passing in a different
     header task parsing function.
 
     Parameters
     ----------
-    constant_name
+    stem_name
         The name for the constant to be defined
 
     metadata_key
@@ -212,6 +310,8 @@ class TaskTimeBudBase(Stem):
         The function used to convert a header into an IP task type
     """
 
+    key_to_petal_dict: dict[str, float]
+
     def __init__(
         self,
         stem_name: str,
@@ -229,8 +329,18 @@ class TaskTimeBudBase(Stem):
         self.ip_task_types = [task.casefold() for task in ip_task_types]
         self.header_parsing_function = header_task_parsing_func
 
-    def setter(self, fits_obj: L0FitsAccess):
-        """Return the desired metadata key only if the parsed task type matches the Bud's task type."""
+    def setter(self, fits_obj: L0FitsAccess) -> float | Type[SpilledDirt]:
+        """
+        Store the metadata key value if the parsed task type is in the desired types.
+
+        Parameters
+        ----------
+        fits_obj
+            The input fits object
+        Returns
+        -------
+        The rounded time
+        """
         task = self.header_parsing_function(fits_obj)
 
         if task.casefold() in self.ip_task_types:
@@ -240,12 +350,22 @@ class TaskTimeBudBase(Stem):
         return SpilledDirt
 
     def getter(self, key: Hashable) -> tuple[float, ...]:
-        """Return a tuple of all the unique values found."""
-        value_tuple = tuple(sorted(set(self.key_to_petal_dict.values())))
-        return value_tuple
+        """
+        Return a tuple of the sorted unique values found.
+
+        Parameters
+        ----------
+        key
+            The input key
+
+        Returns
+        -------
+        A tuple that is the sorted set of unique times
+        """
+        return tuple(sorted(set(self.key_to_petal_dict.values())))
 
 
-class TaskExposureTimesBud(TaskTimeBudBase):
+class TaskExposureTimesBud(TaskRoundTimeBudBase):
     """Produce a tuple of all FPA exposure times present in the dataset for a specific ip task type."""
 
     def __init__(
@@ -262,7 +382,7 @@ class TaskExposureTimesBud(TaskTimeBudBase):
         )
 
 
-class TaskReadoutExpTimesBud(TaskTimeBudBase):
+class TaskReadoutExpTimesBud(TaskRoundTimeBudBase):
     """Produce a tuple of all sensor readout exposure times present in the dataset for a specific task type."""
 
     def __init__(

dkist_processing_common/tasks/mixin/quality/_metrics.py

@@ -1356,13 +1356,15 @@ class _WavecalQualityMixin:
         Note that the residuals are the *unweighed* residuals.
         """
         weight_data = np.ones(input_wavelength.size) if weights is None else weights
-        prepared_weights = fit_result.prepared_weights
+        prepared_weights = np.sqrt(weight_data / np.sum(weight_data))
         residuals = fit_result.minimizer_result.residual / prepared_weights
         residuals[~np.isfinite(residuals)] = 0.0
+        best_fit_atlas = input_spectrum - residuals
         normalized_residuals = residuals / input_spectrum
 
-        best_fit_atlas = fit_result.best_fit_atlas
-        best_fit_wavelength = fit_result.best_fit_wavelength_vector
+        best_fit_header = fit_result.wavelength_parameters.to_header(axis_num=1)
+        wcs = WCS(best_fit_header)
+        best_fit_wavelength = wcs.spectral.pixel_to_world(np.arange(input_spectrum.size))
 
         finite_idx = (
             np.isfinite(input_wavelength)
@@ -1376,7 +1378,7 @@ class _WavecalQualityMixin:
         data = {
             "input_wavelength_nm": input_wavelength.to_value(u.nm)[finite_idx].tolist(),
             "input_spectrum": input_spectrum[finite_idx].tolist(),
-            "best_fit_wavelength_nm": best_fit_wavelength[finite_idx].tolist(),
+            "best_fit_wavelength_nm": best_fit_wavelength.to_value(u.nm)[finite_idx].tolist(),
             "best_fit_atlas": best_fit_atlas[finite_idx].tolist(),
             "normalized_residuals": normalized_residuals[finite_idx].tolist(),
             "weights": None if weights is None else weight_data[finite_idx].tolist(),