dkist-processing-common 11.0.0rc2__py3-none-any.whl → 11.1.0__py3-none-any.whl

dkist_processing_common/manual.py

@@ -12,7 +12,7 @@ from dkist_processing_common.codecs.basemodel import basemodel_encoder
 from dkist_processing_common.models.graphql import RecipeRunProvenanceMutation
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.base import WorkflowTaskBase
-from dkist_processing_common.tests.conftest import FakeGQLClient
+from dkist_processing_common.tests.mock_metadata_store import fake_gql_client_factory
 
 
 logger = logging.getLogger(__name__)
@@ -67,7 +67,7 @@ class ManualProcessing:
 
             with patch(
                 "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
-                new=FakeGQLClient,
+                new=fake_gql_client_factory(),
             ) as foo:
                 # Run the task with a FakeGQLClient. This will handle pre_run(), run(), and post_run()
                 with patch(

dkist_processing_common/models/graphql.py

@@ -1,5 +1,7 @@
 """GraphQL Data models for the metadata store api."""
 from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import field_serializer
 from pydantic import field_validator
 from pydantic import Json
 
@@ -7,20 +9,26 @@ from dkist_processing_common.models.input_dataset import InputDatasetBaseModel
 from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 
 
-class RecipeRunMutation(BaseModel):
+class GraphqlBaseModel(BaseModel):
+    """Custom BaseModel for input datasets."""
+
+    model_config = ConfigDict(validate_assignment=True)
+
+
+class RecipeRunMutation(GraphqlBaseModel):
     """Recipe run mutation record."""
 
     recipeRunId: int
     recipeRunStatusId: int
 
 
-class RecipeRunStatusQuery(BaseModel):
+class RecipeRunStatusQuery(GraphqlBaseModel):
     """Recipe run status query for the recipeRunStatuses endpoint."""
 
     recipeRunStatusName: str
 
 
-class RecipeRunStatusMutation(BaseModel):
+class RecipeRunStatusMutation(GraphqlBaseModel):
     """Recipe run status mutation record."""
 
     recipeRunStatusName: str
@@ -28,13 +36,13 @@ class RecipeRunStatusMutation(BaseModel):
     recipeRunStatusDescription: str
 
 
-class RecipeRunStatusResponse(BaseModel):
+class RecipeRunStatusResponse(GraphqlBaseModel):
     """Response to a recipe run status query."""
 
     recipeRunStatusId: int
 
 
-class InputDatasetPartTypeResponse(BaseModel):
+class InputDatasetPartTypeResponse(GraphqlBaseModel):
     """Response class for the input dataset part type entity."""
 
     inputDatasetPartTypeName: str
@@ -54,13 +62,13 @@ class InputDatasetPartResponse(InputDatasetBaseModel):
         return InputDatasetPartDocumentList(doc_list=value_list)
 
 
-class InputDatasetInputDatasetPartResponse(BaseModel):
+class InputDatasetInputDatasetPartResponse(GraphqlBaseModel):
     """Response class for the join entity between input datasets and input dataset parts."""
 
     inputDatasetPart: InputDatasetPartResponse
 
 
-class InputDatasetResponse(BaseModel):
+class InputDatasetResponse(GraphqlBaseModel):
     """Input dataset query response."""
 
     inputDatasetId: int
@@ -68,33 +76,33 @@ class InputDatasetResponse(BaseModel):
     inputDatasetInputDatasetParts: list[InputDatasetInputDatasetPartResponse]
 
 
-class InputDatasetRecipeInstanceResponse(BaseModel):
+class InputDatasetRecipeInstanceResponse(GraphqlBaseModel):
     """Recipe instance query response."""
 
     inputDataset: InputDatasetResponse
 
 
-class InputDatasetRecipeRunResponse(BaseModel):
+class InputDatasetRecipeRunResponse(GraphqlBaseModel):
     """Recipe run query response."""
 
     recipeInstance: InputDatasetRecipeInstanceResponse
 
 
-class RecipeInstanceResponse(BaseModel):
+class RecipeInstanceResponse(GraphqlBaseModel):
     """Recipe instance query response."""
 
     recipeId: int
     inputDatasetId: int
 
 
-class RecipeRunProvenanceResponse(BaseModel):
+class RecipeRunProvenanceResponse(GraphqlBaseModel):
     """Response for the metadata store recipeRunProvenances and mutations endpoints."""
 
     recipeRunProvenanceId: int
     isTaskManual: bool
 
 
-class RecipeRunConfiguration(BaseModel):
+class RecipeRunConfiguration(GraphqlBaseModel):
     """Response class for a recipe run configuration dictionary."""
 
     validate_l1_on_write: bool = True
@@ -103,10 +111,10 @@ class RecipeRunConfiguration(BaseModel):
     trial_directory_name: str | None = None
     trial_root_directory_name: str | None = None
     teardown_enabled: bool = True
-    trial_exclusive_transfer_tag_lists: list[str] | None = None
+    trial_exclusive_transfer_tag_lists: list[list[str]] | None = None
 
 
-class RecipeRunResponse(BaseModel):
+class RecipeRunResponse(GraphqlBaseModel):
     """Recipe run query response."""
 
     recipeInstance: RecipeInstanceResponse
@@ -122,20 +130,24 @@ class RecipeRunResponse(BaseModel):
             return RecipeRunConfiguration()
         return RecipeRunConfiguration.model_validate(value)
 
+    @field_serializer("configuration")
+    def _serialize_as_basemodel(self, config: RecipeRunConfiguration):
+        return config.model_dump()
+
 
-class RecipeRunMutationResponse(BaseModel):
+class RecipeRunMutationResponse(GraphqlBaseModel):
     """Recipe run mutation response."""
 
     recipeRunId: int
 
 
-class RecipeRunQuery(BaseModel):
+class RecipeRunQuery(GraphqlBaseModel):
     """Query parameters for the metadata store endpoint recipeRuns."""
 
     recipeRunId: int
 
 
-class DatasetCatalogReceiptAccountMutation(BaseModel):
+class DatasetCatalogReceiptAccountMutation(GraphqlBaseModel):
     """
     Dataset catalog receipt account mutation record.
 
@@ -147,13 +159,13 @@ class DatasetCatalogReceiptAccountMutation(BaseModel):
     expectedObjectCount: int
 
 
-class DatasetCatalogReceiptAccountResponse(BaseModel):
+class DatasetCatalogReceiptAccountResponse(GraphqlBaseModel):
     """Dataset catalog receipt account response for query and mutation endpoints."""
 
     datasetCatalogReceiptAccountId: int
 
 
-class RecipeRunProvenanceMutation(BaseModel):
+class RecipeRunProvenanceMutation(GraphqlBaseModel):
     """Recipe run provenance mutation record."""
 
     inputDatasetId: int
@@ -165,7 +177,7 @@ class RecipeRunProvenanceMutation(BaseModel):
     codeVersion: str | None = None
 
 
-class QualityCreation(BaseModel):
+class QualityCreation(GraphqlBaseModel):
     """Quality data creation record."""
 
     datasetId: str
@@ -185,13 +197,13 @@ class QualityCreation(BaseModel):
     efficiencyData: str | None = None
 
 
-class QualitiesRequest(BaseModel):
+class QualitiesRequest(GraphqlBaseModel):
     """Query parameters for quality data."""
 
     datasetId: str
 
 
-class QualityResponse(BaseModel):
+class QualityResponse(GraphqlBaseModel):
     """Query Response for quality data."""
 
     qualityId: int

dkist_processing_common/models/input_dataset.py

@@ -18,7 +18,10 @@ class InputDatasetBaseModel(BaseModel):
     """Custom BaseModel for input datasets."""
 
     model_config = ConfigDict(
-        alias_generator=to_camel, validate_by_name=True, validate_by_alias=True
+        alias_generator=to_camel,
+        validate_by_name=True,
+        validate_by_alias=True,
+        validate_assignment=True,
     )
 
     def model_dump(self, **kwargs) -> dict:

dkist_processing_common/tasks/assemble_movie.py

@@ -2,10 +2,10 @@
 import logging
 from abc import ABC
 from abc import abstractmethod
+from importlib.resources import files
 from typing import Literal
 
 import numpy as np
-import pkg_resources
 from matplotlib import colormaps
 from moviepy import VideoClip
 from PIL import Image
@@ -66,7 +66,7 @@ class AssembleMovie(WorkflowTaskBase, ABC):
     MINIMUM_DURATION = 10  # seconds
     MAXIMUM_DURATION = 60  # seconds
     FPS = 15
-    FONT_FILE = pkg_resources.resource_filename("dkist_processing_common", "fonts/Lato-Regular.ttf")
+    FONT_FILE = files("dkist_processing_common").joinpath("fonts/Lato-Regular.ttf")
     TEXT_MARGIN_PX = 5
     MPL_COLOR_MAP = "viridis"
 

dkist_processing_common/tasks/mixin/quality/_metrics.py

@@ -1365,13 +1365,22 @@ class _WavecalQualityMixin:
         wcs = WCS(best_fit_header)
         best_fit_wavelength = wcs.spectral.pixel_to_world(np.arange(input_spectrum.size))
 
+        finite_idx = (
+            np.isfinite(input_wavelength)
+            * np.isfinite(input_spectrum)
+            * np.isfinite(best_fit_wavelength)
+            * np.isfinite(best_fit_atlas)
+            * np.isfinite(normalized_residuals)
+            * np.isfinite(weight_data)
+        )
+
         data = {
-            "input_wavelength_nm": input_wavelength.to_value(u.nm).tolist(),
-            "input_spectrum": input_spectrum.tolist(),
-            "best_fit_wavelength_nm": best_fit_wavelength.to_value(u.nm).tolist(),
-            "best_fit_atlas": best_fit_atlas.tolist(),
-            "normalized_residuals": normalized_residuals.tolist(),
-            "weights": None if weights is None else weight_data.tolist(),
+            "input_wavelength_nm": input_wavelength.to_value(u.nm)[finite_idx].tolist(),
+            "input_spectrum": input_spectrum[finite_idx].tolist(),
+            "best_fit_wavelength_nm": best_fit_wavelength.to_value(u.nm)[finite_idx].tolist(),
+            "best_fit_atlas": best_fit_atlas[finite_idx].tolist(),
+            "normalized_residuals": normalized_residuals[finite_idx].tolist(),
+            "weights": None if weights is None else weight_data[finite_idx].tolist(),
         }
 
         self._record_values(values=data, tags=[Tag.quality(MetricCode.wavecal_fit)])
@@ -1447,7 +1456,7 @@ class _WavecalQualityMixin:
             name="Wavelength Calibration Results",
             description="These plots show the wavelength solution computed based on fits to a Solar FTS atlas. "
             "The top plot shows the input and best-fit spectra along with the best-fit atlas, which is "
-            "a combination of Solar and Telluric spectra. The bottom plot shows the fir residuals.",
+            "a combination of Solar and Telluric spectra. The bottom plot shows the fit residuals.",
             metric_code=MetricCode.wavecal_fit,
             vertical_multi_pane_plot_data=full_plot,
         )

dkist_processing_common/tasks/trial_output_data.py

@@ -129,9 +129,6 @@ class TransferTrialData(TransferDataBase, GlobusMixin):
         """
         tag_lists = self.transfer_tag_lists
 
-        if not isinstance(tag_lists[0], list):
-            raise ValueError(f"{tag_lists=} must be a list of tag set lists")
-
         transfer_items = []
         for tag_set in tag_lists:
 

dkist_processing_common/tests/conftest.py

@@ -32,21 +32,9 @@ from dkist_processing_pac.optics.telescope import Telescope
 from dkist_processing_common._util.constants import ConstantsDb
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common._util.tags import TagDB
-from dkist_processing_common.models.graphql import InputDatasetInputDatasetPartResponse
-from dkist_processing_common.models.graphql import InputDatasetPartResponse
-from dkist_processing_common.models.graphql import InputDatasetPartTypeResponse
-from dkist_processing_common.models.graphql import InputDatasetRecipeInstanceResponse
-from dkist_processing_common.models.graphql import InputDatasetRecipeRunResponse
-from dkist_processing_common.models.graphql import InputDatasetResponse
-from dkist_processing_common.models.graphql import RecipeInstanceResponse
-from dkist_processing_common.models.graphql import RecipeRunProvenanceResponse
-from dkist_processing_common.models.graphql import RecipeRunResponse
-from dkist_processing_common.models.graphql import RecipeRunStatusResponse
-from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.tasks import WorkflowTaskBase
-
-TILE_SIZE = 64
+from dkist_processing_common.tests.mock_metadata_store import fake_gql_client
 
 
 @pytest.fixture()
@@ -99,6 +87,21 @@ def constants_db(recipe_run_id) -> ConstantsDb:
     constants.close()
 
 
+@pytest.fixture()
+def fake_constants_db() -> dict:
+    """
+    A fake constants DB to prevent key errors.
+
+    Usage on a task: task.constants._update(fake_constants_db)
+    """
+    db = {
+        "PROPOSAL_ID": "PROPID",
+        "INSTRUMENT": "INSTRUMENT",
+        "OBS_IP_START_TIME": "20240416T160000",
+    }
+    return db
+
+
 class CommonDataset(Spec122Dataset):
     def __init__(self, polarimetric: bool = True):
         super().__init__(
@@ -285,18 +288,18 @@ def cs_step_angle_round_ndigits() -> int:
 
 
 @pytest.fixture(scope="session")
-def angle_random_max_perturabtion(cs_step_angle_round_ndigits) -> float:
+def angle_random_max_perturbation(cs_step_angle_round_ndigits) -> float:
     # Ensures that we always round down to zero.
     # E.g., if ndigits = 1 then this value will be 0.049.
     return 10**-cs_step_angle_round_ndigits / 2 - 10 ** -(cs_step_angle_round_ndigits + 2)
 
 
 @pytest.fixture(scope="session")
-def grouped_cal_sequence_headers(angle_random_max_perturabtion) -> dict[int, list[L0FitsAccess]]:
+def grouped_cal_sequence_headers(angle_random_max_perturbation) -> dict[int, list[L0FitsAccess]]:
     ds = CalibrationSequenceDataset(
         array_shape=(1, 2, 2),
         time_delta=2.0,
-        angle_max_random_perturbation=angle_random_max_perturabtion,
+        angle_max_random_perturbation=angle_random_max_perturbation,
     )
     header_list = [
         spec122_validator.validate_and_translate_to_214_l0(d.header(), return_type=fits.HDUList)[
@@ -331,155 +334,9 @@ def max_cs_step_time_sec() -> float:
     return 20.0
 
 
-class FakeGQLClient:
-
-    observe_frames_doc_object = [
-        {
-            "bucket": uuid4().hex[:6],
-            "object_keys": [Path(uuid4().hex[:6]).as_posix() for _ in range(3)],
-        }
-    ]
-
-    calibration_frames_doc_object = [
-        {
-            "bucket": uuid4().hex[:6],
-            "object_keys": [Path(uuid4().hex[:6]).as_posix() for _ in range(3)],
-        },
-        {
-            "bucket": uuid4().hex[:6],
-            "object_keys": [Path(uuid4().hex[:6]).as_posix() for _ in range(3)],
-        },
-    ]
-
-    parameters_doc_object = [
-        {
-            "parameterName": "param_name_1",
-            "parameterValues": [
-                {
-                    "parameterValueId": 1,
-                    "parameterValue": json.dumps([[1, 2, 3], [4, 5, 6], [7, 8, 9]]),
-                    "parameterValueStartDate": datetime(2000, 1, 1).isoformat(),
-                }
-            ],
-        },
-        {
-            "parameterName": "param_name_2",
-            "parameterValues": [
-                {
-                    "parameterValueId": 2,
-                    "parameterValue": json.dumps(
-                        {
-                            "__file__": {
-                                "bucket": "data",
-                                "objectKey": f"parameters/param_name/{uuid4().hex}.dat",
-                            }
-                        }
-                    ),
-                    "parameterValueStartDate": datetime(2000, 1, 1).isoformat(),
-                },
-                {
-                    "parameterValueId": 3,
-                    "parameterValue": json.dumps(
-                        {
-                            "__file__": {
-                                "bucket": "data",
-                                "objectKey": f"parameters/param_name/{uuid4().hex}.dat",
-                            }
-                        }
-                    ),
-                    "parameterValueStartDate": datetime(2000, 1, 2).isoformat(),
-                },
-            ],
-        },
-        {
-            "parameterName": "param_name_4",
-            "parameterValues": [
-                {
-                    "parameterValueId": 4,
-                    "parameterValue": json.dumps(
-                        {"a": 1, "b": 3.14159, "c": "foo", "d": [1, 2, 3]}
-                    ),
-                    "parameterValueStartDate": datetime(2000, 1, 1).isoformat(),
-                }
-            ],
-        },
-    ]
-
-    def __init__(self, *args, **kwargs):
-        pass
-
-    def execute_gql_query(self, **kwargs):
-        query_base = kwargs["query_base"]
-        if query_base == "recipeRunStatuses":
-            return [RecipeRunStatusResponse(recipeRunStatusId=1)]
-        if query_base == "recipeRuns":
-            if kwargs.get("query_response_cls") == InputDatasetRecipeRunResponse:
-                return [
-                    InputDatasetRecipeRunResponse(
-                        recipeInstance=InputDatasetRecipeInstanceResponse(
-                            inputDataset=InputDatasetResponse(
-                                inputDatasetId=1,
-                                isActive=True,
-                                inputDatasetInputDatasetParts=[
-                                    InputDatasetInputDatasetPartResponse(
-                                        inputDatasetPart=InputDatasetPartResponse(
-                                            inputDatasetPartId=1,
-                                            inputDatasetPartDocument=json.dumps(
-                                                self.parameters_doc_object
-                                            ),
-                                            inputDatasetPartType=InputDatasetPartTypeResponse(
-                                                inputDatasetPartTypeName="parameters"
-                                            ),
-                                        )
-                                    ),
-                                    InputDatasetInputDatasetPartResponse(
-                                        inputDatasetPart=InputDatasetPartResponse(
-                                            inputDatasetPartId=2,
-                                            inputDatasetPartDocument=json.dumps(
-                                                self.observe_frames_doc_object
-                                            ),
-                                            inputDatasetPartType=InputDatasetPartTypeResponse(
-                                                inputDatasetPartTypeName="observe_frames"
-                                            ),
-                                        )
-                                    ),
-                                    InputDatasetInputDatasetPartResponse(
-                                        inputDatasetPart=InputDatasetPartResponse(
-                                            inputDatasetPartId=3,
-                                            inputDatasetPartDocument=json.dumps(
-                                                self.calibration_frames_doc_object
-                                            ),
-                                            inputDatasetPartType=InputDatasetPartTypeResponse(
-                                                inputDatasetPartTypeName="calibration_frames"
-                                            ),
-                                        )
-                                    ),
-                                ],
-                            ),
-                        ),
-                    ),
-                ]
-
-            return [
-                RecipeRunResponse(
-                    recipeInstanceId=1,
-                    recipeInstance=RecipeInstanceResponse(
-                        recipeId=1,
-                        inputDatasetId=1,
-                    ),
-                    configuration=f'{{"tile_size": {TILE_SIZE}}}',
-                    recipeRunProvenances=[
-                        RecipeRunProvenanceResponse(recipeRunProvenanceId=1, isTaskManual=False),
-                    ],
-                ),
-            ]
-
-    @staticmethod
-    def execute_gql_mutation(**kwargs):
-        ...
-
-
-# All the following stuff is copied from dkist-processing-pac
+####################################
+# Copied from dkist-processing-pac #
+####################################
 def compute_telgeom(time_hst: Time):
     dkist_lon = (156 + 15 / 60.0 + 21.7 / 3600.0) * (-1)
     dkist_lat = 20 + 42 / 60.0 + 27.0 / 3600.0
@@ -543,7 +400,7 @@ class CalibrationSequenceStepDataset(Spec122Dataset):
         return "none" if self.pol_status == "clear" else str(self.pol_theta)
 
     @key_function("PAC__006")
-    def retarter_status(self, key: str) -> str:
+    def retarder_status(self, key: str) -> str:
         return self.ret_status
 
     @key_function("PAC__007")
@@ -795,6 +652,11 @@ def post_fit_polcal_fitter(
     return fitter
 
 
+#################
+# Input Dataset #
+#################
+
+
 class InputDatasetTask(WorkflowTaskBase):
     def run(self):
         pass
@@ -826,58 +688,3 @@ def task_with_input_dataset(
             file_path.write_text(data=json.dumps({"doc_list": part}))
             task.tag(path=file_path, tags=tag)
         yield task
-
-
-def create_parameter_files(
-    task: WorkflowTaskBase, parameters_doc: list[dict] = FakeGQLClient.parameters_doc_object
-):
-    """
-    Create the parameter files specified in the parameters document returned by the metadata store.
-
-    This fixture assumes that the JSON parameters document has already been loaded into a python
-    structure, but the parameter values themselves are still JSON.
-    """
-    for parameter in parameters_doc:
-        for value in parameter["parameterValues"]:
-            if "__file__" not in value["parameterValue"]:
-                continue
-            parameter_value = json.loads(value["parameterValue"])
-            param_path = parameter_value["__file__"]["objectKey"]
-            file_path = task.scratch.workflow_base_path / Path(param_path)
-            if not file_path.parent.exists():
-                file_path.parent.mkdir(parents=True, exist_ok=True)
-            file_path.write_text(data="")
-            task.tag(path=file_path, tags=Tag.parameter(param_path))
-
-
-def create_input_frames(
-    task: WorkflowTaskBase,
-    input_frame_docs: list[dict] = FakeGQLClient.observe_frames_doc_object
-    + FakeGQLClient.calibration_frames_doc_object,
-):
-    """
-    Create the observe and calibration frame files specified in the input dataset documents
-    returned by the metadata store.
-    """
-    for frame in input_frame_docs:
-        for object_key in frame["object_keys"]:
-            file_path = task.scratch.workflow_base_path / Path(object_key)
-            if not file_path.parent.exists():
-                file_path.parent.mkdir(parents=True, exist_ok=True)
-            file_path.write_text(data="")
-            task.tag(path=file_path, tags=[Tag.frame(), Tag.input()])
-
-
-@pytest.fixture()
-def fake_constants_db() -> dict:
-    """
-    A fake constants DB to prevent key errors.
-
-    Usage on a task: task.constants._update(fake_constants_db)
-    """
-    db = {
-        "PROPOSAL_ID": "PROPID",
-        "INSTRUMENT": "INSTRUMENT",
-        "OBS_IP_START_TIME": "20240416T160000",
-    }
-    return db