dkist-processing-common 10.5.4__py3-none-any.whl → 12.1.0rc1__py3-none-any.whl

dkist_processing_common/models/message.py

@@ -1,11 +1,11 @@
 """Data structures for messages placed on the interservice bus."""
+
 from typing import Type
 
 from pydantic import Field
 from talus import MessageBodyBase
 from talus import PublishMessageBase
 
-
 ########################
 # Message Body Schemas #
 ########################

dkist_processing_common/models/message_queue_binding.py

@@ -1,4 +1,5 @@
 """Binding between a queue and a message to be published."""
+
 from talus import Binding
 from talus import Queue
 
@@ -7,7 +8,6 @@ from dkist_processing_common.models.message import CatalogFrameMessage
 from dkist_processing_common.models.message import CatalogObjectMessage
 from dkist_processing_common.models.message import CreateQualityReportMessage
 
-
 catalog_frame_queue = Queue(
     name="catalog.frame.q", arguments=common_configurations.isb_queue_arguments
 )

dkist_processing_common/models/metric_code.py

@@ -1,4 +1,5 @@
 """Controlled list of quality metric codes."""
+
 from enum import StrEnum
 
 
@@ -23,3 +24,4 @@ class MetricCode(StrEnum):
     range = "RANGE"
     sensitivity = "SENSITIVITY"
     task_types = "TASK_TYPES"
+    wavecal_fit = "WAVECAL_FIT"

dkist_processing_common/models/parameters.py

@@ -1,14 +1,23 @@
 """Base class for parameter-parsing object."""
+
 import logging
+from contextlib import contextmanager
 from datetime import datetime
+from pathlib import Path
 from typing import Any
+from typing import Callable
 from typing import Literal
 
 import numpy as np
 import scipy.interpolate as spi
-from astropy.io import fits
 
-from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetParameterValue
+from dkist_processing_common._util.scratch import WorkflowFileSystem
+from dkist_processing_common.codecs.array import array_decoder
+from dkist_processing_common.codecs.basemodel import basemodel_decoder
+from dkist_processing_common.codecs.fits import fits_array_decoder
+from dkist_processing_common.models.input_dataset import InputDatasetFilePointer
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
+from dkist_processing_common.models.tags import Tag
 
 logger = logging.getLogger(__name__)
 
@@ -24,9 +33,9 @@ class ParameterBase:
 
     To use in an instrument pipeline a subclass is required. Here's a simple, but complete example::
 
-        class InstParameters(ParameterBase)
-            def __init__(self, input_dataset_parameters, some_other_parameter):
-                super().__init__(input_dataset_parameters)
+        class InstParameters(ParameterBase):
+            def __init__(self, scratch, some_other_parameters):
+                super().__init__(scratch=scratch)
                 self._thing = self._some_function(some_other_parameters)
 
             @property
@@ -34,7 +43,7 @@ class ParameterBase:
                 return self._find_most_recent_past_value("some_parameter_name")
 
             @property
-            def complicate_parameter(self):
+            def complicated_parameter(self):
                 return self._some_complicated_parsing_function("complicated_parameter_name", another_argument)
 
 
@@ -55,15 +64,16 @@ class ParameterBase:
                     workflow_version=workflow_version,
                 )
 
-                self.parameters = InstParameters(self.input_dataset_parameters)  #<------ This is the important line
+                self.parameters = InstParameters(scratch=self.scratch)  #<------ This is the important line
 
-    Note that the first argument to the ConstantsSubclass with *always* be self.input_dataset_parameters, but
-    additional argument can be passed if the subclass requires them.
+    ParameterBase needs the task scratch in order to read the parameters document written at input dataset
+    transfer.  Note that the first argument to the ConstantsSubclass will *always* be scratch, but additional
+    arguments can be passed if the subclass requires them.
 
     Parameters
     ----------
-    input_dataset_parameters
-        The input parameters
+     scratch
+        The task scratch WorkflowFileSystem instance
 
     obs_ip_start_time
         A string containing the start date of the Observe IP task type frames. Must be in isoformat.
@@ -74,25 +84,53 @@ class ParameterBase:
 
     def __init__(
         self,
-        input_dataset_parameters: dict[str, list[InputDatasetParameterValue]],
+        scratch: WorkflowFileSystem,
         obs_ip_start_time: str | None = None,
         **kwargs,
     ):
+        self.scratch = scratch
+        input_dataset_parameter_model = self._get_parameters_doc_from_file()
+        input_dataset_parameters = {}
+        if input_dataset_parameter_model is not None:
+            input_dataset_parameters = {
+                p.parameter_name: p.parameter_values for p in input_dataset_parameter_model.doc_list
+            }
         self.input_dataset_parameters = input_dataset_parameters
+
         if obs_ip_start_time is not None:
             # Specifically `not None` because we want to error normally on badly formatted strings (including "").
             self._obs_ip_start_datetime = datetime.fromisoformat(obs_ip_start_time)
         else:
             logger.info(
                 "WARNING: "
-                "The task containing this parameters object did not provide an obs ip start time. "
-                "This really only makes sense for Parsing tasks."
+                "The task containing this parameters object did not provide an obs ip start time, "
+                "which really only makes sense for Parsing tasks."
             )
 
         for parent_class in self.__class__.__bases__:
             if hasattr(parent_class, "is_param_mixin"):
                 parent_class.__init__(self, **kwargs)
 
+    def _read_parameter_file(
+        self, tag: str, decoder: Callable[[Path], Any], **decoder_kwargs
+    ) -> Any:
+        """Read any file in the task scratch instance."""
+        paths = list(self.scratch.find_all(tags=tag))
+        if len(paths) == 0:
+            logger.info(f"WARNING: There is no parameter file for {tag = }")
+        if len(paths) == 1:
+            return decoder(paths[0], **decoder_kwargs)
+        if len(paths) > 1:
+            raise ValueError(f"There is more than one parameter file for {tag = }: {paths}")
+
+    def _get_parameters_doc_from_file(self) -> InputDatasetPartDocumentList:
+        """Get parameters doc saved at the TransferL0Data task."""
+        tag = Tag.input_dataset_parameters()
+        parameters_from_file = self._read_parameter_file(
+            tag=tag, decoder=basemodel_decoder, model=InputDatasetPartDocumentList
+        )
+        return parameters_from_file
+
     def _find_most_recent_past_value(
         self,
         parameter_name: str,
@@ -113,20 +151,19 @@ class ParameterBase:
             )
         return result
 
-    @staticmethod
-    def _load_param_value_from_fits(param_dict: dict, hdu: int = 0) -> np.ndarray:
-        """Load a numpy array from a parameter pointing to a FITS file."""
-        file_path = param_dict["param_path"]
-
-        hdul = fits.open(file_path)
-        return hdul[hdu].data
-
-    @staticmethod
-    def _load_param_value_from_numpy_save(param_dict: dict) -> np.ndarray:
-        """Return the data associated with a parameter file saved in numpy format."""
-        file_path = param_dict["param_path"]
-        result = np.load(file_path)
-        return result
+    def _load_param_value_from_fits(
+        self, param_obj: InputDatasetFilePointer, hdu: int = 0
+    ) -> np.ndarray:
+        """Return the data associated with a tagged parameter file saved in FITS format."""
+        tag = param_obj.file_pointer.tag
+        param_value = self._read_parameter_file(tag=tag, decoder=fits_array_decoder, hdu=hdu)
+        return param_value
+
+    def _load_param_value_from_numpy_save(self, param_obj: InputDatasetFilePointer) -> np.ndarray:
+        """Return the data associated with a tagged parameter file saved in numpy format."""
+        tag = param_obj.file_pointer.tag
+        param_value = self._read_parameter_file(tag=tag, decoder=array_decoder)
+        return param_value
 
 
 class _ParamMixinBase:

dkist_processing_common/models/quality.py

@@ -1,7 +1,11 @@
 """Support classes used to create a quality report."""
+
 from typing import Any
 
 from pydantic import BaseModel
+from pydantic import Field
+from pydantic import field_validator
+from pydantic_core.core_schema import ValidationInfo
 
 
 class Plot2D(BaseModel):
@@ -13,6 +17,50 @@ class Plot2D(BaseModel):
     series_name: str | None = None
     ylabel_horizontal: bool = False
     ylim: tuple[float, float] | None = None
+    plot_kwargs: dict[str, dict[str, Any]] = Field(default_factory=dict)
+    sort_series: bool = True
+
+
+class VerticalMultiPanePlot2D(BaseModel):
+    """
+    Support class to hold a multi-pane plot with plots stacked vertically.
+
+    This type of metric is really geared towards plots that share an X axis and have no gap between them. If you just
+    want two separate plots it's probably better to use a list of `Plot2D` objects.
+    """
+
+    top_to_bottom_plot_list: list[Plot2D]
+    match_x_axes: bool = True
+    no_gap: bool = True
+    top_to_bottom_height_ratios: list[float] | None = None
+
+    @field_validator("top_to_bottom_height_ratios")
+    @classmethod
+    def ensure_same_number_of_height_ratios_and_plots(
+        cls, height_ratios: list[float] | None, info: ValidationInfo
+    ) -> list[float]:
+        """
+        Make sure that the number of height ratios is the same as the number of plots.
+
+        Also populates default, same-size ratios if no ratios were given.
+        """
+        try:
+            plot_list = info.data["top_to_bottom_plot_list"]
+        except KeyError:
+            # The plot list didn't validate for some reason. We're about to error anyway.
+            return [1.0]
+
+        num_plots = len(plot_list)
+        if height_ratios is None:
+            return [1.0] * num_plots
+
+        if len(height_ratios) != num_plots:
+            raise ValueError(
+                f"The number of items in `top_to_bottom_height_ratios` list ({len(height_ratios)}) is not "
+                f"the same as the number of plots ({num_plots})"
+            )
+
+        return height_ratios
 
 
 class SimpleTable(BaseModel):
@@ -57,11 +105,7 @@ class PlotRaincloud(BaseModel):
 
 
 class ReportMetric(BaseModel):
-    """
-    A Quality Report is made up of a list of metrics with the schema defined by this class.
-
-    Additionally, this class can produce a Flowable or List of Flowables to be render the metric in the PDF Report
-    """
+    """A Quality Report is made up of a list of metrics with the schema defined by this class."""
 
     name: str
     description: str
@@ -69,6 +113,7 @@ class ReportMetric(BaseModel):
     facet: str | None = None
     statement: str | list[str] | None = None
     plot_data: Plot2D | list[Plot2D] | None = None
+    multi_plot_data: VerticalMultiPanePlot2D | None = None
     histogram_data: PlotHistogram | list[PlotHistogram] | None = None
     table_data: SimpleTable | list[SimpleTable] | None = None
     modmat_data: ModulationMatrixHistograms | None = None

dkist_processing_common/models/tags.py

@@ -1,5 +1,7 @@
 """Components of the Tag model.  Stem + Optional Suffix = Tag."""
+
 from enum import Enum
+from enum import StrEnum
 
 from dkist_processing_common.models.task_name import TaskName
 
@@ -7,7 +9,7 @@ from dkist_processing_common.models.task_name import TaskName
 EXP_TIME_ROUND_DIGITS: int = 6
 
 
-class StemName(str, Enum):
+class StemName(StrEnum):
     """Controlled list of Tag Stems."""
 
     output = "OUTPUT"
@@ -42,7 +44,7 @@ class Tag:
     """Controlled methods for creating tags from stems + optional suffixes."""
 
     @staticmethod
-    def format_tag(stem: StemName | str, *parts):
+    def format_tag(stem: StemName | str, *parts) -> str:
         """
         Create a formatted tag sting given the input parts.
 
@@ -63,7 +65,7 @@ class Tag:
 
     # Static Tags
     @classmethod
-    def movie_frame(cls):
+    def movie_frame(cls) -> str:
         """
         Return a movie frame tag.
 
@@ -74,7 +76,7 @@ class Tag:
         return cls.format_tag(StemName.movie_frame)
 
     @classmethod
-    def input(cls):
+    def input(cls) -> str:
         """
         Return an input tag.
 
@@ -96,7 +98,7 @@ class Tag:
         return cls.format_tag(StemName.calibrated)
 
     @classmethod
-    def output(cls):
+    def output(cls) -> str:
         """
         Return an output tag.
 
@@ -107,7 +109,7 @@ class Tag:
         return cls.format_tag(StemName.output)
 
     @classmethod
-    def frame(cls):
+    def frame(cls) -> str:
         """
         Return a frame tag.
 
@@ -118,7 +120,7 @@ class Tag:
         return cls.format_tag(StemName.frame)
 
     @classmethod
-    def intermediate(cls):
+    def intermediate(cls) -> str:
         """
         Return an intermediate tag.
 
@@ -129,7 +131,7 @@ class Tag:
         return cls.format_tag(StemName.intermediate)
 
     @classmethod
-    def input_dataset_observe_frames(cls):
+    def input_dataset_observe_frames(cls) -> str:
         """
         Return an input dataset observe frames tag.
 
@@ -140,7 +142,7 @@ class Tag:
         return cls.format_tag(StemName.input_dataset, "observe_frames")
 
     @classmethod
-    def input_dataset_calibration_frames(cls):
+    def input_dataset_calibration_frames(cls) -> str:
         """
         Return an input dataset calibration frames tag.
 
@@ -151,7 +153,7 @@ class Tag:
         return cls.format_tag(StemName.input_dataset, "calibration_frames")
 
     @classmethod
-    def input_dataset_parameters(cls):
+    def input_dataset_parameters(cls) -> str:
         """
         Return an input dataset parameters tag.
 
@@ -162,7 +164,7 @@ class Tag:
         return cls.format_tag(StemName.input_dataset, "parameters")
 
     @classmethod
-    def movie(cls):
+    def movie(cls) -> str:
         """
         Return a movie tag.
 
@@ -230,7 +232,7 @@ class Tag:
 
     @classmethod
     def task_demodulation_matrices(cls) -> str:
-        """Tags intermediate demodulation matric calibration objects."""
+        """Tags intermediate demodulation matrix calibration objects."""
         return cls.task(TaskName.demodulation_matrices.value)
 
     @classmethod
@@ -254,13 +256,13 @@ class Tag:
         return cls.task(TaskName.geometric_offsets.value)
 
     @classmethod
-    def task_geometric_sepectral_shifts(cls) -> str:
+    def task_geometric_spectral_shifts(cls) -> str:
         """Tags intermediate geometric spectral shift calibration objects."""
         return cls.task(TaskName.geometric_spectral_shifts.value)
 
     # Dynamic Tags
     @classmethod
-    def task(cls, ip_task_type: str):
+    def task(cls, ip_task_type: str) -> str:
         """
         Return a task tag for the given task type.
 
@@ -275,7 +277,7 @@ class Tag:
         return cls.format_tag(StemName.task, ip_task_type)
 
     @classmethod
-    def cs_step(cls, n: int):
+    def cs_step(cls, n: int) -> str:
         """
         Return a cs step tag for the given cs_step number.
 
@@ -291,7 +293,7 @@ class Tag:
         return cls.format_tag(StemName.cs_step, n)
 
     @classmethod
-    def modstate(cls, n: int):
+    def modstate(cls, n: int) -> str:
         """
         Return a modstate tag for the given modstate number.
 
@@ -323,7 +325,7 @@ class Tag:
         return cls.format_tag(StemName.stokes, stokes_state)
 
     @classmethod
-    def dsps_repeat(cls, dsps_repeat_number: int):
+    def dsps_repeat(cls, dsps_repeat_number: int) -> str:
         """
         Return a dsps repeat tag for the given dsps_repeat number.
 
@@ -428,18 +430,18 @@ class Tag:
         return cls.format_tag(StemName.workflow_task, class_name)
 
     @classmethod
-    def dataset_inventory(cls):
+    def dataset_inventory(cls) -> str:
         """
-        Return an dataset_inventory tag.
+        Return a dataset_inventory tag.
 
         Returns
         -------
-        An dataset_inventory tag
+        A dataset_inventory tag
         """
         return cls.format_tag(StemName.dataset_inventory)
 
     @classmethod
-    def asdf(cls):
+    def asdf(cls) -> str:
         """
         Return an asdf tag.
 

dkist_processing_common/models/task_name.py

@@ -1,8 +1,9 @@
 """Controlled list of common IP task tag names."""
-from enum import Enum
 
+from enum import StrEnum
 
-class TaskName(str, Enum):
+
+class TaskName(StrEnum):
     """Controlled list of task tag names."""
 
     observe = "OBSERVE"

dkist_processing_common/models/telemetry.py

@@ -0,0 +1,28 @@
+"""Models to support telemetry data."""
+
+from pydantic import BaseModel
+
+
+class ObservableProgress(BaseModel, validate_assignment=True):
+    """Container for tracking progress for a metering instrument e.g. task progress."""
+
+    current: int = 0
+    total: int = 0
+
+    def increment(self, step: int = 1) -> None:
+        """Increment the current progress by the given step."""
+        self.current += step
+
+    @property
+    def percent_complete(self) -> float:
+        """Return the percent complete as a float between 0 and 100."""
+        if self.total > 0:
+            return (self.current / self.total) * 100
+        return 0.0
+
+    def set_complete(self):
+        """Set the current progress to the total."""
+        if self.total == 0:
+            self.total = self.current = 1
+        else:
+            self.current = self.total

dkist_processing_common/models/wavelength.py

@@ -1,10 +1,11 @@
 """Support classes for manipulating wavelengths."""
+
 import astropy.units as u
 from pydantic import BaseModel
 from pydantic import ConfigDict
+from pydantic import ValidationInfo
 from pydantic import field_validator
 from pydantic import model_validator
-from pydantic import ValidationInfo
 
 
 class WavelengthRange(BaseModel):
@@ -26,3 +27,4 @@ class WavelengthRange(BaseModel):
         """Validate that the max wavelength is greater than the min wavelength."""
         if self.min > self.max:
             raise ValueError("min is greater than max.  Values may be reversed.")
+        return self

dkist_processing_common/parsers/average_bud.py

@@ -0,0 +1,46 @@
+"""Pre-made flower that reads a single header key from all files with specific task types and returns the average."""
+
+from enum import StrEnum
+from statistics import mean
+from typing import Callable
+from typing import Hashable
+
+import numpy as np
+
+from dkist_processing_common.parsers.near_bud import TaskNearFloatBud
+from dkist_processing_common.parsers.task import passthrough_header_ip_task
+
+
+class TaskAverageBud(TaskNearFloatBud):
+    """
+    Pre-made bud that returns the average of a single header key from all files with specific task types.
+
+    Parameters
+    ----------
+    constant_name
+        The name for the constant to be defined
+
+    metadata_key
+        The metadata key associated with the constant
+
+    ip_task_types
+        Only consider objects whose parsed header IP task type matches a string in this list
+
+    task_type_parsing_function
+        The function used to convert a header into an IP task type
+    """
+
+    def __init__(
+        self,
+        constant_name: str,
+        metadata_key: str | StrEnum,
+        ip_task_types: str | list[str],
+        task_type_parsing_function: Callable = passthrough_header_ip_task,
+    ):
+        super().__init__(
+            constant_name=constant_name,
+            metadata_key=metadata_key,
+            ip_task_types=ip_task_types,
+            tolerance=np.inf,
+            task_type_parsing_function=task_type_parsing_function,
+        )

dkist_processing_common/parsers/cs_step.py

@@ -1,4 +1,5 @@
 """Classes supporting Calibration Sequence steps."""
+
 from __future__ import annotations
 
 from datetime import datetime
@@ -6,6 +7,7 @@ from datetime import timezone
 from typing import Type
 
 from dkist_processing_common.models.constants import BudName
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.tags import StemName
@@ -100,8 +102,9 @@ class CSStepFlower(Stem):
     """
 
     def __init__(self, max_cs_step_time_sec: float):
-        super().__init__(stem_name=StemName.cs_step.value)
+        super().__init__(stem_name=StemName.cs_step)
         self.max_cs_step_time_sec = max_cs_step_time_sec
+        self.CS_step_set = set()
 
     def setter(self, fits_obj: L0FitsAccess) -> CSStep | Type[SpilledDirt]:
         """
@@ -118,7 +121,10 @@ class CSStepFlower(Stem):
         """
         if fits_obj.ip_task_type != "polcal":
             return SpilledDirt
-        return CSStep(fits_obj, max_cs_time_sec=self.max_cs_step_time_sec)
+
+        cs_step = CSStep(fits_obj, max_cs_time_sec=self.max_cs_step_time_sec)
+        self.CS_step_set.add(cs_step)
+        return cs_step
 
     def getter(self, key) -> str | float | int:
         """
@@ -132,11 +138,11 @@ class CSStepFlower(Stem):
         -------
         The cs step for the given key
         """
-        unique_steps = sorted(list(set(self.key_to_petal_dict.values())))
+        unique_steps = sorted(self.CS_step_set)
         return unique_steps.index(self.key_to_petal_dict[key])
 
 
-class NumCSStepBud(Stem):
+class NumCSStepBud(SetStem):
     """
     The total number of CS Steps present in a dataset.
 
@@ -147,7 +153,7 @@ class NumCSStepBud(Stem):
     """
 
     def __init__(self, max_cs_step_time_sec: float):
-        super().__init__(stem_name=BudName.num_cs_steps.value)
+        super().__init__(stem_name=BudName.num_cs_steps)
         self.max_cs_step_time_sec = max_cs_step_time_sec
 
     def setter(self, fits_obj: L0FitsAccess) -> CSStep | Type[SpilledDirt]:
@@ -167,17 +173,12 @@ class NumCSStepBud(Stem):
             return SpilledDirt
         return CSStep(fits_obj, max_cs_time_sec=self.max_cs_step_time_sec)
 
-    def getter(self, key) -> int:
+    def getter(self) -> int:
         """
         Return the number of CS Steps present.
 
-        Parameters
-        ----------
-        key
-            The input key
         Returns
         -------
         The number of cs steps associated with the key
         """
-        value_set = set(self.key_to_petal_dict.values())
-        return len(value_set)
+        return len(self.value_set)

dkist_processing_common/parsers/dsps_repeat.py

@@ -1,5 +1,7 @@
 """Classes supporting the Data Set Parameters Set (DSPS) Repeat parameter."""
+
 from dkist_processing_common.models.constants import BudName
+from dkist_processing_common.models.fits_access import MetadataKey
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.models.tags import StemName
 from dkist_processing_common.models.task_name import TaskName
@@ -15,9 +17,9 @@ class TotalDspsRepeatsBud(TaskUniqueBud):
 
     def __init__(self):
         super().__init__(
-            constant_name=BudName.num_dsps_repeats.value,
-            metadata_key="num_dsps_repeats",
-            ip_task_type=TaskName.observe.value,
+            constant_name=BudName.num_dsps_repeats,
+            metadata_key=MetadataKey.num_dsps_repeats,
+            ip_task_types=TaskName.observe,
         )
 
 
@@ -26,7 +28,7 @@ class DspsRepeatNumberFlower(SingleValueSingleKeyFlower):
 
     def __init__(self):
         super().__init__(
-            tag_stem_name=StemName.dsps_repeat.value, metadata_key="current_dsps_repeat"
+            tag_stem_name=StemName.dsps_repeat, metadata_key=MetadataKey.current_dsps_repeat
         )
 
     def setter(self, fits_obj: L0FitsAccess):