dkist-processing-common 10.5.4__py3-none-any.whl → 12.1.0rc1__py3-none-any.whl

dkist_processing_common/models/flower_pot.py

@@ -1,11 +1,16 @@
 """
 Framework for grouping multiple keys and values with arbitrary logic.
 
-Defines:
-    Stem -> ABC for groupings that depend on both the key and (maybe) value. Subgroups (Petals) are implied but not enforced.
+The key components are:
+    **Stem:** ABC for groupings that depend on both the key and (maybe) value. Subgroups (Petals) are implied but not enforced.
 
-    FlowerPot -> Container for Stem children (Flowers)
+    **ListStem:** ABC for groups that depend on value only. More limited, but faster than `Stem` for cases where the keys don't matter.
+
+    **SetStem:** ABC for groups that depend on value only and the values are well represented by a `set`. Even more limited, but faster than `Stem` for cases where the keys don't matter.
+
+    **FlowerPot:** Container for Stem children (Flowers)
 """
+
 from __future__ import annotations
 
 from abc import ABC
@@ -16,7 +21,7 @@ from typing import Any
 
 
 class FlowerPot:
-    """Base class to hold multiple sets (stems) of key, value pairs."""
+    """Base class to hold multiple sets (Stems) of key, value pairs."""
 
     def __init__(self):
         self.stems: list[Stem] = list()
@@ -54,9 +59,9 @@ class FlowerPot:
 
 class SpilledDirt:
     """
-    A custom class for when a Flower wants the FlowerPot to skip that particular key/value.
+    A custom class for when a Stem wants the FlowerPot to skip that particular key/value.
 
-    Exists because None, False, [], (), etc. etc. are all valid Flower return values
+    Exists because None, False, [], (), etc. etc. are all valid Stem return values
     """
 
 
@@ -89,7 +94,7 @@ class Petal:
 
 class Stem(ABC):
     """
-    Base group for grouping keys via arbitrary logic on the total collection of keys and values.
+    Base class for grouping keys via arbitrary logic on the total collection of keys and values.
 
     Parameters
     ----------
@@ -132,6 +137,15 @@ class Stem(ABC):
 
         return self._petal_cache
 
+    @property
+    def can_be_picked(self) -> bool:
+        """
+        Return True if there are any values to be picked.
+
+        A `Stem` could have no values even after dirt is added if all of the results were `SpilledDirt`.
+        """
+        return len(self.petals) > 0
+
     def _generate_petal_list(self) -> None:
         """
         Generate a list of petals.
@@ -145,7 +159,7 @@ class Stem(ABC):
         changes, `key_to_petal_dict`, is unhashable.
         """
         petal_to_key_dict = defaultdict(list)
-        for key, petal in self.key_to_petal_dict.items():
+        for key in self.key_to_petal_dict.keys():
             petal = self.getter(key)
             petal_to_key_dict[petal].append(key)
 
@@ -179,7 +193,7 @@ class Stem(ABC):
     @abstractmethod
     def getter(self, key: Hashable) -> Hashable:
         """
-        Logic to apply to all ingested values when picking the Flower.
+        Logic to apply to all ingested values when picking the Stem.
 
         Implemented in derived class.
 
@@ -192,3 +206,211 @@ class Stem(ABC):
         The value
         """
         pass
+
+
+class ListStem(ABC):
+    """
+    Base class for collecting and applying logic to values in a `list` with a `Stem`-like interface.
+
+    Unlike the full `Stem`, this class does NOT retain information about the keys and thus does no grouping of keys based
+    on values. The direct consequence of this is that the `.petals` property is undefined and will raise an ``AttributeError``
+    if accessed. This also means there is no need to invert the `key_to_petal_dict` (because it doesn't exist), which,
+    in turn, means there is no need to run the `getter` for every key. The result is that the `bud` property only needs
+    one call to `getter`. Thus, the calculation of a single value derived from all values (i.e., `bud`) is much faster
+    than using a full `Stem`.
+
+    Parameters
+    ----------
+    stem_name
+        The name to be associated with the stem
+    """
+
+    def __init__(self, stem_name: Any):
+        self.stem_name = stem_name
+        self.value_list: list = list()
+        self._need_to_compute_bud_value: bool = True
+
+    def update(self, key: Any, value: Any) -> None:
+        """
+        Ingest a single key/value pair. Note that the ``key`` is not used.
+
+        Parameters
+        ----------
+        key
+            The key (unused)
+
+        value
+            The value
+
+        Returns
+        -------
+        None
+        """
+        result = self.setter(value)
+        if result is not SpilledDirt:
+            self.value_list.append(result)
+            self._need_to_compute_bud_value = True
+
+    @property
+    def petals(self) -> None:
+        """Raise an error because `ListStem` does not retain key information and therefore cannot group keys."""
+        raise AttributeError(
+            f"{self.__class__.__name__} subclasses ListStem and therefore does not define the `petals` property"
+        )
+
+    @property
+    def can_be_picked(self) -> bool:
+        """
+        Return True if there are any values to be picked.
+
+        A `Stem` could have no values even after dirt is added if all of the results were `SpilledDirt`.
+        """
+        return len(self.value_list) > 0
+
+    @property
+    def bud(self) -> Petal:
+        """Return the result of `getter` packaged in a `Petal` object."""
+        if self._need_to_compute_bud_value:
+            self._value_cache = self.getter()
+            self._need_to_compute_bud_value = False
+
+        return Petal((self._value_cache, "LISTSTEM_NOT_USED"))
+
+    @abstractmethod
+    def setter(self, value: Any) -> Any:
+        """
+        Logic to apply to a single value pair on ingest.
+
+        Implemented in derived class.
+
+        Parameters
+        ----------
+        value
+            The value to be added
+
+        Returns
+        -------
+        Any
+        """
+        pass
+
+    @abstractmethod
+    def getter(self) -> Any:
+        """
+        Logic to apply to all ingested values when computing the `bud`.
+
+        Implemented in derived class.
+
+        Returns
+        -------
+        The value of the bud
+        """
+        pass
+
+
+class SetStem(ABC):
+    """
+    Base class for collecting and applying logic to values in a `set` with a `Stem`-like interface.
+
+    Unlike the full `Stem`, this class does NOT retain information about the keys and thus does no grouping of keys based
+    on values. The direct consequence of this is that the `.petals` property is undefined and will raise an ``AttributeError``
+    if accessed. This also means there is no need to invert the `key_to_petal_dict` (because it doesn't exist), which,
+    in turn, means there is no need to run the `getter` for every key. The result is that the `bud` property only needs
+    one call to `getter`. Combined with the efficiency of storing values in a `set`, the calculation of a single value
+    derived from all values (i.e., `bud`) is much faster than using a full `Stem`.
+
+    .. Note::
+      The use of a `set` as the underlying storage mechanism means information regarding how many times a particular value
+      is present will be lost. It also means the return type of `setter` must be hashable. Both of these constraints can
+      be avoided by using `ListStem`, which still gets a significant speedup over `Stem` by dropping key information.
+
+
+    Parameters
+    ----------
+    stem_name
+        The name to be associated with the stem
+    """
+
+    def __init__(self, stem_name: Any):
+        self.stem_name = stem_name
+        self.value_set: set = set()
+        self._need_to_compute_bud_value: bool = True
+
+    def update(self, key: Any, value: Any) -> None:
+        """
+        Ingest a single key/value pair. Note that the ``key`` is not used.
+
+        Parameters
+        ----------
+        key
+            The key (unused)
+
+        value
+            The value
+
+        Returns
+        -------
+        None
+        """
+        result = self.setter(value)
+        if result is not SpilledDirt:
+            self.value_set.add(result)
+            self._need_to_compute_bud_value = True
+
+    @property
+    def petals(self) -> None:
+        """Raise an error because `SetStem` does not retain key information and therefore cannot group keys."""
+        raise AttributeError(
+            f"{self.__class__.__name__} subclasses SetStem and therefore does not define the `petals` property"
+        )
+
+    @property
+    def can_be_picked(self) -> bool:
+        """
+        Return True if there are any values to be picked.
+
+        A `Stem` could have no values even after dirt is added if all of the results were `SpilledDirt`.
+        """
+        return len(self.value_set) > 0
+
+    @property
+    def bud(self) -> Petal:
+        """Return the result of `getter` packaged in a `Petal` object."""
+        if self._need_to_compute_bud_value:
+            self._value_cache = self.getter()
+            self._need_to_compute_bud_value = False
+
+        return Petal((self._value_cache, "SETSTEM_NOT_USED"))
+
+    @abstractmethod
+    def setter(self, value: Any) -> Hashable:
+        """
+        Logic to apply to a single value pair on ingest.
+
+        Must return a Hashable object because the result will be stored in a `set`.
+
+        Implemented in derived class.
+
+        Parameters
+        ----------
+        value
+            The value to be added
+
+        Returns
+        -------
+        Any
+        """
+        pass
+
+    @abstractmethod
+    def getter(self) -> Any:
+        """
+        Logic to apply to all ingested values when computing the `bud`.
+
+        Implemented in derived class.
+
+        Returns
+        -------
+        The value of the bud
+        """
+        pass

dkist_processing_common/models/fried_parameter.py

@@ -0,0 +1,41 @@
+"""Helper methods to handle fried parameter / r0 validity."""
+
+
+def r0_valid(
+    r0: float | None = None,
+    ao_lock: bool | None = None,
+    num_out_of_bounds_ao_values: int | None = None,
+) -> bool:
+    """
+    Determine if the r0 value should be considered valid based on the following conditions.
+
+        * ATMOS_R0 does not exist in the header.
+        * the value of ATMOS_R0 is greater than 0.3m
+        * the AO is not locked
+        * the value of OOBSHIFT is greater than 100
+
+    When the adaptive optics system is not locked, the ATMOS_R0 keyword is still filled with the output of the
+    Fried parameter calculation. The inputs are not valid in this instance and the value should be removed.
+
+    Sometimes, due to timing differences between the calculation of the Fried parameter and the AO lock status being
+    updated, non-physical values can be recorded for ATMOS_R0 right on the edge of an AO_LOCK state change. To
+    combat this, any remaining R0 values greater than 30cm (which is beyond the realm of physical possibility for
+    solar observations) are also removed.
+
+    In addition, the number of AO out-of-bound values is given in the keyword OOBSHIFT and the AO team advises
+    that values under 100 are when the r0 value is considered reliable. If the OOBSHIFT key doesn't exist, this check
+    should be ignored.
+    """
+    if r0 is None:
+        return False
+
+    if r0 > 0.3:
+        return False
+
+    if ao_lock is not True:
+        return False
+
+    if num_out_of_bounds_ao_values is not None and num_out_of_bounds_ao_values > 100:
+        return False
+
+    return True

dkist_processing_common/models/graphql.py

@@ -1,24 +1,35 @@
 """GraphQL Data models for the metadata store api."""
-from dataclasses import dataclass
 
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import Json
+from pydantic import field_serializer
+from pydantic import field_validator
 
-@dataclass
-class RecipeRunMutation:
+from dkist_processing_common.models.input_dataset import InputDatasetBaseModel
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
+
+
+class GraphqlBaseModel(BaseModel):
+    """Custom BaseModel for input datasets."""
+
+    model_config = ConfigDict(validate_assignment=True)
+
+
+class RecipeRunMutation(GraphqlBaseModel):
     """Recipe run mutation record."""
 
     recipeRunId: int
     recipeRunStatusId: int
 
 
-@dataclass
-class RecipeRunStatusQuery:
+class RecipeRunStatusQuery(GraphqlBaseModel):
     """Recipe run status query for the recipeRunStatuses endpoint."""
 
     recipeRunStatusName: str
 
 
-@dataclass
-class RecipeRunStatusMutation:
+class RecipeRunStatusMutation(GraphqlBaseModel):
     """Recipe run status mutation record."""
 
     recipeRunStatusName: str
@@ -26,38 +37,39 @@ class RecipeRunStatusMutation:
     recipeRunStatusDescription: str
 
 
-@dataclass
-class RecipeRunStatusResponse:
+class RecipeRunStatusResponse(GraphqlBaseModel):
     """Response to a recipe run status query."""
 
     recipeRunStatusId: int
 
 
-@dataclass
-class InputDatasetPartTypeResponse:
+class InputDatasetPartTypeResponse(GraphqlBaseModel):
     """Response class for the input dataset part type entity."""
 
     inputDatasetPartTypeName: str
 
 
-@dataclass
-class InputDatasetPartResponse:
+class InputDatasetPartResponse(InputDatasetBaseModel):
     """Response class for the input dataset part entity."""
 
     inputDatasetPartId: int
-    inputDatasetPartDocument: str
+    # inputDatasetPartDocument : Json[InputDatasetPartDocumentList] # will work in gqlclient v2
+    inputDatasetPartDocument: Json[list]
     inputDatasetPartType: InputDatasetPartTypeResponse
 
+    @field_validator("inputDatasetPartDocument", mode="after")
+    @classmethod
+    def _use_frame_or_parameter_model(cls, value_list):  # not needed for gqlclient v2
+        return InputDatasetPartDocumentList(doc_list=value_list)
 
-@dataclass
-class InputDatasetInputDatasetPartResponse:
+
+class InputDatasetInputDatasetPartResponse(GraphqlBaseModel):
     """Response class for the join entity between input datasets and input dataset parts."""
 
     inputDatasetPart: InputDatasetPartResponse
 
 
-@dataclass
-class InputDatasetResponse:
+class InputDatasetResponse(GraphqlBaseModel):
     """Input dataset query response."""
 
     inputDatasetId: int
@@ -65,62 +77,78 @@ class InputDatasetResponse:
     inputDatasetInputDatasetParts: list[InputDatasetInputDatasetPartResponse]
 
 
-@dataclass
-class InputDatasetRecipeInstanceResponse:
+class InputDatasetRecipeInstanceResponse(GraphqlBaseModel):
     """Recipe instance query response."""
 
     inputDataset: InputDatasetResponse
 
 
-@dataclass
-class InputDatasetRecipeRunResponse:
+class InputDatasetRecipeRunResponse(GraphqlBaseModel):
     """Recipe run query response."""
 
     recipeInstance: InputDatasetRecipeInstanceResponse
 
 
-@dataclass
-class RecipeInstanceResponse:
+class RecipeInstanceResponse(GraphqlBaseModel):
     """Recipe instance query response."""
 
     recipeId: int
     inputDatasetId: int
 
 
-@dataclass
-class RecipeRunProvenanceResponse:
+class RecipeRunProvenanceResponse(GraphqlBaseModel):
     """Response for the metadata store recipeRunProvenances and mutations endpoints."""
 
     recipeRunProvenanceId: int
     isTaskManual: bool
 
 
-@dataclass
-class RecipeRunResponse:
+class RecipeRunConfiguration(GraphqlBaseModel):
+    """Response class for a recipe run configuration dictionary."""
+
+    validate_l1_on_write: bool = True
+    destination_bucket: str = "data"
+    tile_size: int | None = None
+    trial_directory_name: str | None = None
+    trial_root_directory_name: str | None = None
+    teardown_enabled: bool = True
+    trial_exclusive_transfer_tag_lists: list[list[str]] | None = None
+
+
+class RecipeRunResponse(GraphqlBaseModel):
     """Recipe run query response."""
 
     recipeInstance: RecipeInstanceResponse
     recipeInstanceId: int
     recipeRunProvenances: list[RecipeRunProvenanceResponse]
-    configuration: str = None
+    # configuration: Json[RecipeRunConfiguration] | None # will work in gqlclient v2
+    configuration: Json[dict] | None
+
+    @field_validator("configuration", mode="after")
+    @classmethod
+    def _use_recipe_run_configuration_model(cls, value):  # not needed for gqlclient v2
+        if value is None:
+            return RecipeRunConfiguration()
+        return RecipeRunConfiguration.model_validate(value)
 
+    @field_serializer("configuration")
+    def _serialize_as_basemodel(self, config: RecipeRunConfiguration):
+        return config.model_dump()
 
-@dataclass
-class RecipeRunMutationResponse:
+
+class RecipeRunMutationResponse(GraphqlBaseModel):
     """Recipe run mutation response."""
 
     recipeRunId: int
 
 
-@dataclass
-class RecipeRunQuery:
+class RecipeRunQuery(GraphqlBaseModel):
     """Query parameters for the metadata store endpoint recipeRuns."""
 
     recipeRunId: int
 
 
-@dataclass
-class DatasetCatalogReceiptAccountMutation:
+class DatasetCatalogReceiptAccountMutation(GraphqlBaseModel):
     """
     Dataset catalog receipt account mutation record.
 
@@ -132,15 +160,13 @@ class DatasetCatalogReceiptAccountMutation:
     expectedObjectCount: int
 
 
-@dataclass
-class DatasetCatalogReceiptAccountResponse:
+class DatasetCatalogReceiptAccountResponse(GraphqlBaseModel):
     """Dataset catalog receipt account response for query and mutation endpoints."""
 
     datasetCatalogReceiptAccountId: int
 
 
-@dataclass
-class RecipeRunProvenanceMutation:
+class RecipeRunProvenanceMutation(GraphqlBaseModel):
     """Recipe run provenance mutation record."""
 
     inputDatasetId: int
@@ -149,39 +175,4 @@ class RecipeRunProvenanceMutation:
     taskName: str
     libraryVersions: str
     workflowVersion: str
-    codeVersion: str = None
-
-
-@dataclass
-class QualityCreation:
-    """Quality data creation record."""
-
-    datasetId: str
-    metricCode: str
-    facet: str | None = None
-    name: str | None = None
-    description: str | None = None
-    statement: str | None = None
-    # JSON array
-    warnings: str | None = None
-    # JSON objects
-    plotData: str | None = None
-    tableData: str | None = None
-    histogramData: str | None = None
-    modmatData: str | None = None
-    raincloudData: str | None = None
-    efficiencyData: str | None = None
-
-
-@dataclass
-class QualitiesRequest:
-    """Query parameters for quality data."""
-
-    datasetId: str
-
-
-@dataclass
-class QualityResponse:
-    """Query Response for quality data."""
-
-    qualityId: int
+    codeVersion: str | None = None

dkist_processing_common/models/input_dataset.py

@@ -0,0 +1,117 @@
+"""Input dataset models for the inputDatasetPartDocument from the metadata store api."""
+
+import json
+from datetime import datetime
+from typing import Any
+
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import Field
+from pydantic import Json
+from pydantic import PlainSerializer
+from pydantic import field_serializer
+from pydantic import field_validator
+from pydantic.alias_generators import to_camel
+from typing_extensions import Annotated
+
+
+class InputDatasetBaseModel(BaseModel):
+    """Custom BaseModel for input datasets."""
+
+    model_config = ConfigDict(
+        alias_generator=to_camel,
+        validate_by_name=True,
+        validate_by_alias=True,
+        validate_assignment=True,
+    )
+
+    def model_dump(self, **kwargs) -> dict:
+        """Dump models as they were in the metadata store."""
+        kwargs.setdefault("exclude_defaults", True)
+        kwargs.setdefault("by_alias", True)  # will not be needed in Pydantic v3
+        return super().model_dump(**kwargs)
+
+    def model_dump_json(self, **kwargs) -> str:
+        """Dump models as they were in the metadata store."""
+        kwargs.setdefault("exclude_defaults", True)
+        kwargs.setdefault("by_alias", True)  # will not be needed in Pydantic v3
+        return super().model_dump_json(**kwargs)
+
+
+class InputDatasetObject(InputDatasetBaseModel):
+    """Input dataset object validator for a single file."""
+
+    bucket: str
+    object_key: str
+    tag: str | None = None
+
+
+class InputDatasetFilePointer(InputDatasetBaseModel):
+    """Wrapper for InputDatasetObject files."""
+
+    file_pointer: InputDatasetObject = Field(alias="__file__")
+
+
+class InputDatasetParameterValue(InputDatasetBaseModel):
+    """Input dataset parameter value validator."""
+
+    parameter_value_id: int
+    # parameter_value: Json[InputDatasetFilePointer] | Json[Any] # will work in gqlclient v2
+    parameter_value: Json[Any]
+    parameter_value_start_date: Annotated[
+        datetime, Field(default=datetime(1, 1, 1)), PlainSerializer(lambda x: x.isoformat())
+    ]
+
+    @field_validator("parameter_value", mode="after")
+    @classmethod
+    def validate_parameter_value(cls, param_val):
+        """Decode and provide additional validation for parameter_value types."""
+        match param_val:
+            case {"__file__": _}:
+                return InputDatasetFilePointer.model_validate(param_val)
+            case _:
+                return param_val
+
+    @field_serializer("parameter_value")
+    def serialize_parameter_value(self, param_val):
+        """Serialize the parameter_value types."""
+        if isinstance(param_val, InputDatasetBaseModel):
+            return json.dumps(param_val.model_dump())
+        return json.dumps(param_val)
+
+
+class InputDatasetParameter(InputDatasetBaseModel):
+    """Parsing of the inputDatasetPartDocument that is relevant for parameters."""
+
+    parameter_name: str
+    parameter_values: list[InputDatasetParameterValue]
+
+    @property
+    def input_dataset_objects(self) -> list[InputDatasetObject]:
+        """Find and return list of InputDatasetObjects."""
+        object_list = []
+        for param in self.parameter_values:
+            if isinstance(param.parameter_value, InputDatasetFilePointer):
+                object_list.append(param.parameter_value.file_pointer)
+        return object_list
+
+
+class InputDatasetFrames(InputDatasetBaseModel):
+    """Parsing of the inputDatasetPartDocument that is relevant for frames."""
+
+    bucket: str
+    object_keys: list[str] = Field(alias="object_keys")  # not camel case in metadata store
+
+    @property
+    def input_dataset_objects(self) -> list[InputDatasetObject]:
+        """Convert a single bucket and a list of object_keys list into a list of InputDatasetObjects."""
+        object_list = []
+        for frame in self.object_keys:
+            object_list.append(InputDatasetObject(bucket=self.bucket, object_key=frame))
+        return object_list
+
+
+class InputDatasetPartDocumentList(InputDatasetBaseModel):
+    """List of either InputDatasetFrames or InputDatasetParameter objects."""
+
+    doc_list: list[InputDatasetFrames] | list[InputDatasetParameter] = Field(alias="doc_list")