dkist-processing-common 10.6.1rc2__tar.gz → 10.6.1rc4__tar.gz

dkist_processing_common-10.6.1rc4/PKG-INFO

@@ -0,0 +1,175 @@
+Metadata-Version: 2.2
+Name: dkist-processing-common
+Version: 10.6.1rc4
+Summary: Common task classes used by the DKIST science data processing pipelines
+Author-email: NSO / AURA <dkistdc@nso.edu>
+License: BSD-3-Clause
+Project-URL: Homepage, https://nso.edu/dkist/data-center/
+Project-URL: Repository, https://bitbucket.org/dkistdc/dkist-processing-common/
+Project-URL: Documentation, https://docs.dkist.nso.edu/projects/common
+Project-URL: Help, https://nso.atlassian.net/servicedesk/customer/portal/5
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.11
+Description-Content-Type: text/x-rst
+Requires-Dist: asdf<4.0.0,>=3.5.0
+Requires-Dist: astropy<7.0.0,>=6.0.0
+Requires-Dist: dkist-fits-specifications<5.0,>=4.0.0
+Requires-Dist: dkist-header-validator<6.0,>=5.0.0
+Requires-Dist: dkist-processing-core==5.1.0
+Requires-Dist: dkist-processing-pac<4.0,>=3.1
+Requires-Dist: dkist-service-configuration<3.0,>=2.0.2
+Requires-Dist: dkist-spectral-lines<4.0,>=3.0.0
+Requires-Dist: globus-sdk>=3.12.0
+Requires-Dist: gqlclient[pydantic]==1.2.3
+Requires-Dist: sqids==0.5.1
+Requires-Dist: matplotlib>=3.4
+Requires-Dist: moviepy>=2.0.0
+Requires-Dist: numpy>=1.20.2
+Requires-Dist: object-clerk==0.1.1
+Requires-Dist: pandas>=1.4.2
+Requires-Dist: pillow>=10.2.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: redis==4.6.0
+Requires-Dist: requests>=2.23
+Requires-Dist: scipy>=1.15.1
+Requires-Dist: sunpy>=3.0.0
+Requires-Dist: talus==1.1.0
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
+Requires-Dist: hypothesis; extra == "test"
+Requires-Dist: towncrier; extra == "test"
+Requires-Dist: dkist-data-simulator>=5.0.0; extra == "test"
+Requires-Dist: dkist-processing-common[inventory]; extra == "test"
+Requires-Dist: dkist-processing-common[asdf]; extra == "test"
+Requires-Dist: dkist-processing-common[quality]; extra == "test"
+Provides-Extra: docs
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: sphinx-astropy; extra == "docs"
+Requires-Dist: sphinx-changelog; extra == "docs"
+Requires-Dist: sphinx-autoapi!=3.1.0; extra == "docs"
+Requires-Dist: pytest; extra == "docs"
+Requires-Dist: towncrier<22.12.0; extra == "docs"
+Requires-Dist: dkist-sphinx-theme; extra == "docs"
+Provides-Extra: inventory
+Requires-Dist: dkist-inventory<2.0,>=1.6.0; extra == "inventory"
+Provides-Extra: asdf
+Requires-Dist: dkist-inventory[asdf]<2.0,>=1.6.0; extra == "asdf"
+Provides-Extra: quality
+Requires-Dist: dkist-quality<2.0,>=1.2.1; extra == "quality"
+
+dkist-processing-common
+=======================
+
+This repository works in concert with `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ and `dkist-processing-*instrument*` to
+form the DKIST calibration processing stack.
+
+Usage
+-----
+
+The classes in this repository should be used as the base of any DKIST processing pipeline tasks. Science tasks should subclass `ScienceTaskL0ToL1Base`.
+
+Each class is built on an abstract base class with the `run` method left for a developer to fill out with the required steps that the task should take.
+This class is then used as the callable object for the workflow and scheduling engine.
+
+Example
+-------
+
+.. code-block:: python
+
+    from dkist_processing_common.tasks.base import ScienceTaskL0ToL1Base
+
+
+    class RemoveArtifacts(ScienceTaskL0ToL1Base):
+        def run(self):
+            # task code here
+            total = 2 + 5
+
+Deployment
+----------
+
+dkist-processing-common is deployed to `PyPI <https://pypi.org/project/dkist-processing-common/>`_
+
+Development
+-----------
+
+There are two prerequisites for test execution on a local machine:
+
+
+* Redis.  A running instance of redis on the local machine is required. The tests will use the default host ip of localhost and port of 6379 to connect to the database.
+
+* RabbitMQ.  A running instance of rabbitmq on the local machine is required.  The tests will use the default host of localhost and a port of 5672 to connect to the interservice bus.
+
+
+To run the tests locally, clone the repository and install the package in editable mode with the test extras.
+
+
+.. code-block:: bash
+
+    git clone git@bitbucket.org:dkistdc/dkist-processing-common.git
+    cd dkist-processing-common
+    pre-commit install
+    pip install -e .[test]
+    # Redis must be running
+    pytest -v --cov dkist_processing_common
+
+Changelog
+#########
+
+When you make **any** change to this repository it **MUST** be accompanied by a changelog file.
+The changelog for this repository uses the `towncrier <https://github.com/twisted/towncrier>`__ package.
+Entries in the changelog for the next release are added as individual files (one per change) to the ``changelog/`` directory.
+
+Writing a Changelog Entry
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A changelog entry accompanying a change should be added to the ``changelog/`` directory.
+The name of a file in this directory follows a specific template::
+
+  <PULL REQUEST NUMBER>.<TYPE>[.<COUNTER>].rst
+
+The fields have the following meanings:
+
+* ``<PULL REQUEST NUMBER>``: This is the number of the pull request, so people can jump from the changelog entry to the diff on BitBucket.
+* ``<TYPE>``: This is the type of the change and must be one of the values described below.
+* ``<COUNTER>``: This is an optional field, if you make more than one change of the same type you can append a counter to the subsequent changes, i.e. ``100.bugfix.rst`` and ``100.bugfix.1.rst`` for two bugfix changes in the same PR.
+
+The list of possible types is defined the the towncrier section of ``pyproject.toml``, the types are:
+
+* ``feature``: This change is a new code feature.
+* ``bugfix``: This is a change which fixes a bug.
+* ``doc``: A documentation change.
+* ``removal``: A deprecation or removal of public API.
+* ``misc``: Any small change which doesn't fit anywhere else, such as a change to the package infrastructure.
+
+
+Rendering the Changelog at Release Time
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When you are about to tag a release first you must run ``towncrier`` to render the changelog.
+The steps for this are as follows:
+
+* Run `towncrier build --version vx.y.z` using the version number you want to tag.
+* Agree to have towncrier remove the fragments.
+* Add and commit your changes.
+* Tag the release.
+
+**NOTE:** If you forget to add a Changelog entry to a tagged release (either manually or automatically with ``towncrier``)
+then the Bitbucket pipeline will fail. To be able to use the same tag you must delete it locally and on the remote branch:
+
+.. code-block:: bash
+
+    # First, actually update the CHANGELOG and commit the update
+    git commit
+
+    # Delete tags
+    git tag -d vWHATEVER.THE.VERSION
+    git push --delete origin vWHATEVER.THE.VERSION
+
+    # Re-tag with the same version
+    git tag vWHATEVER.THE.VERSION
+    git push --tags origin main

dkist_processing_common-10.6.1rc4/changelog/236.misc.1.rst

@@ -0,0 +1,2 @@
+Change returns from the metadata store queries into Pydantic BaseModel instances.  Remove unnecessary parsing
+and error checking in the metadata store mixin.

dkist_processing_common-10.6.1rc4/changelog/236.misc.rst

@@ -0,0 +1,3 @@
+Convert dataclasses in the graphql model to Pydantic BaseModels for additional validation. In the
+RecipeRunResponse class, configuration is converted from a JSON dictionary to its own Pydantic BaseModel.
+In the InputDatasetPartResponse class, the inputDatasetPartDocument is now returned as a list of dictionaries.

{dkist_processing_common-10.6.1rc2 → dkist_processing_common-10.6.1rc4}/dkist_processing_common/models/graphql.py

@@ -1,5 +1,6 @@
 """GraphQL Data models for the metadata store api."""
 from pydantic import BaseModel
+from pydantic import field_validator
 from pydantic import Json
 
 
@@ -84,13 +85,32 @@ class RecipeRunProvenanceResponse(BaseModel):
     isTaskManual: bool
 
 
+class RecipeRunConfiguration(BaseModel):
+    """Response class for a recipe run configuration dictionary."""
+
+    validate_l1_on_write: bool = True
+    destination_bucket: str = "data"
+    tile_size: int | None = None
+    trial_directory_name: str | None = None
+    trial_root_directory_name: str | None = None
+    teardown_enabled: bool = True
+    trial_exclusive_transfer_tag_lists: list[str] | None = None
+
+
 class RecipeRunResponse(BaseModel):
     """Recipe run query response."""
 
     recipeInstance: RecipeInstanceResponse
     recipeInstanceId: int
     recipeRunProvenances: list[RecipeRunProvenanceResponse]
-    configuration: Json[dict] | None = {}
+    configuration: Json[RecipeRunConfiguration] | None
+
+    @field_validator("configuration", mode="after")
+    @classmethod
+    def _use_default_configuration_model(cls, value):
+        if value is None:
+            return RecipeRunConfiguration()
+        return value
 
 
 class RecipeRunMutationResponse(BaseModel):
@@ -132,7 +152,7 @@ class RecipeRunProvenanceMutation(BaseModel):
     taskName: str
     libraryVersions: str
     workflowVersion: str
-    codeVersion: str = None
+    codeVersion: str | None = None
 
 
 class QualityCreation(BaseModel):

{dkist_processing_common-10.6.1rc2 → dkist_processing_common-10.6.1rc4}/dkist_processing_common/tasks/mixin/metadata_store.py

@@ -4,6 +4,8 @@ import logging
 from functools import cached_property
 from typing import Literal
 
+from pydantic import validate_call
+
 from dkist_processing_common._util.graphql import GraphQLClient
 from dkist_processing_common.codecs.quality import QualityDataEncoder
 from dkist_processing_common.config import common_configurations
@@ -61,6 +63,7 @@ class MetadataStoreMixin:
         if len(response) > 0:
             return response[0].recipeRunStatusId
 
+    @validate_call
     def _metadata_store_create_recipe_run_status(self, status: str, is_complete: bool) -> int:
         """
         Add a new recipe run status to the db.
@@ -75,10 +78,6 @@ class MetadataStoreMixin:
             "marked complete.",
         }
 
-        if not isinstance(status, str):
-            raise TypeError(f"status must be of type str: {status}")
-        if not isinstance(is_complete, bool):
-            raise TypeError(f"is_complete must be of type bool: {is_complete}")
         params = RecipeRunStatusMutation(
             recipeRunStatusName=status,
             isComplete=is_complete,
@@ -197,7 +196,7 @@ class MetadataStoreMixin:
     # INPUT DATASET RECIPE RUN
 
     @cached_property
-    def metadata_store_input_dataset_recipe_run_response(self) -> InputDatasetRecipeRunResponse:
+    def metadata_store_input_dataset_recipe_run(self) -> InputDatasetRecipeRunResponse:
         """Get the input dataset recipe run response from the metadata store."""
         params = RecipeRunQuery(recipeRunId=self.recipe_run_id)
         response = self.metadata_store_client.execute_gql_query(
@@ -213,7 +212,7 @@ class MetadataStoreMixin:
         """Get the input dataset part by input dataset part type name."""
         part_type_dict = {}
         parts = (
-            self.metadata_store_input_dataset_recipe_run_response.recipeInstance.inputDataset.inputDatasetInputDatasetParts
+            self.metadata_store_input_dataset_recipe_run.recipeInstance.inputDataset.inputDatasetInputDatasetParts
         )
         for part in parts:
             part_type_name = part.inputDatasetPart.inputDatasetPartType.inputDatasetPartTypeName

{dkist_processing_common-10.6.1rc2 → dkist_processing_common-10.6.1rc4}/dkist_processing_common/tasks/output_data_base.py

@@ -19,7 +19,7 @@ class OutputDataBase(WorkflowTaskBase, ABC):
     @cached_property
     def destination_bucket(self) -> str:
         """Get the destination bucket."""
-        return self.metadata_store_recipe_run.configuration.get("destination_bucket", "data")
+        return self.metadata_store_recipe_run.configuration.destination_bucket
 
     def format_object_key(self, path: Path) -> str:
         """

{dkist_processing_common-10.6.1rc2 → dkist_processing_common-10.6.1rc4}/dkist_processing_common/tasks/teardown.py

@@ -22,7 +22,7 @@ class TeardownBase(WorkflowTaskBase, ABC):
     @property
     def teardown_enabled(self) -> bool:
         """Recipe run configuration indicating if data should be removed at the end of a run."""
-        return self.metadata_store_recipe_run.configuration.get("teardown_enabled", True)
+        return self.metadata_store_recipe_run.configuration.teardown_enabled
 
     def run(self) -> None:
         """Run method for Teardown class."""

{dkist_processing_common-10.6.1rc2 → dkist_processing_common-10.6.1rc4}/dkist_processing_common/tasks/transfer_input_data.py

@@ -2,6 +2,7 @@
 import logging
 from pathlib import Path
 
+from dkist_processing_common.codecs.json import json_encoder
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.base import WorkflowTaskBase
 from dkist_processing_common.tasks.mixin.globus import GlobusMixin
@@ -20,11 +21,11 @@ class TransferL0Data(WorkflowTaskBase, GlobusMixin, InputDatasetMixin):
     def download_input_dataset(self):
         """Get the input dataset document parts and save it to scratch with the appropriate tags."""
         if doc := self.metadata_store_input_dataset_observe_frames.inputDatasetPartDocument:
-            self.write(doc.encode("utf-8"), tags=Tag.input_dataset_observe_frames())
+            self.write(doc, tags=Tag.input_dataset_observe_frames(), encoder=json_encoder)
         if doc := self.metadata_store_input_dataset_calibration_frames.inputDatasetPartDocument:
-            self.write(doc.encode("utf-8"), tags=Tag.input_dataset_calibration_frames())
+            self.write(doc, tags=Tag.input_dataset_calibration_frames(), encoder=json_encoder)
         if doc := self.metadata_store_input_dataset_parameters.inputDatasetPartDocument:
-            self.write(doc.encode("utf-8"), tags=Tag.input_dataset_parameters())
+            self.write(doc, tags=Tag.input_dataset_parameters(), encoder=json_encoder)
 
     def format_transfer_items(
         self, input_dataset_objects: list[InputDatasetObject]

{dkist_processing_common-10.6.1rc2 → dkist_processing_common-10.6.1rc4}/dkist_processing_common/tasks/trial_output_data.py

@@ -43,25 +43,23 @@ class TransferTrialData(TransferDataBase, GlobusMixin):
 
     @cached_property
     def destination_bucket(self) -> str:
-        """Get the destination bucket with a trial default."""
-        return self.metadata_store_recipe_run.configuration.get("destination_bucket", "etc")
+        """Get the destination bucket."""
+        return self.metadata_store_recipe_run.configuration.destination_bucket
 
     @property
     def destination_root_folder(self) -> Path:
         """Format the destination root folder with a value that can be set in the recipe run configuration."""
-        root_name_from_configuration = self.metadata_store_recipe_run.configuration.get(
-            "trial_root_directory_name"
+        root_name_from_config = (
+            self.metadata_store_recipe_run.configuration.trial_root_directory_name
         )
-        root_name = Path(root_name_from_configuration or super().destination_root_folder)
-
+        root_name = Path(root_name_from_config or super().destination_root_folder)
         return root_name
 
     @property
     def destination_folder(self) -> Path:
         """Format the destination folder with a parent that can be set by the recipe run configuration."""
-        dir_name = self.metadata_store_recipe_run.configuration.get("trial_directory_name") or Path(
-            self.constants.dataset_id
-        )
+        dir_name_from_config = self.metadata_store_recipe_run.configuration.trial_directory_name
+        dir_name = dir_name_from_config or Path(self.constants.dataset_id)
         return self.destination_root_folder / dir_name
 
     @property
@@ -71,9 +69,12 @@ class TransferTrialData(TransferDataBase, GlobusMixin):
         Defaults to transferring all product files.  Setting `trial_exclusive_transfer_tag_lists` in the
         recipe run configuration to a list of tag lists will override the default.
         """
-        return self.metadata_store_recipe_run.configuration.get(
-            "trial_exclusive_transfer_tag_lists", self.default_transfer_tag_lists
+        tag_list_from_config = (
+            self.metadata_store_recipe_run.configuration.trial_exclusive_transfer_tag_lists
         )
+        if tag_list_from_config is not None:
+            return tag_list_from_config
+        return self.default_transfer_tag_lists
 
     @property
     def output_frame_tag_list(self) -> list[list[str]]:

{dkist_processing_common-10.6.1rc2 → dkist_processing_common-10.6.1rc4}/dkist_processing_common/tasks/write_l1.py

@@ -105,14 +105,14 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
                     spec214_validator.validate(self.scratch.absolute_path(relative_path))
 
     @cached_property
-    def tile_size_param(self) -> int:
+    def tile_size_param(self) -> int | None:
         """Get the tile size parameter for compression."""
-        return self.metadata_store_recipe_run.configuration.get("tile_size", None)
+        return self.metadata_store_recipe_run.configuration.tile_size
 
     @cached_property
     def validate_l1_on_write(self) -> bool:
         """Check for validate on write."""
-        return self.metadata_store_recipe_run.configuration.get("validate_l1_on_write", True)
+        return self.metadata_store_recipe_run.configuration.validate_l1_on_write
 
     @cached_property
     def workflow_had_manual_intervention(self):

{dkist_processing_common-10.6.1rc2 → dkist_processing_common-10.6.1rc4}/dkist_processing_common/tests/conftest.py

@@ -333,6 +333,79 @@ def max_cs_step_time_sec() -> float:
 
 
 class FakeGQLClient:
+
+    observe_frames_doc_object = [
+        {
+            "bucket": uuid4().hex[:6],
+            "object_keys": [Path(uuid4().hex[:6]).as_posix() for _ in range(3)],
+        }
+    ]
+
+    calibration_frames_doc_object = [
+        {
+            "bucket": uuid4().hex[:6],
+            "object_keys": [Path(uuid4().hex[:6]).as_posix() for _ in range(3)],
+        },
+        {
+            "bucket": uuid4().hex[:6],
+            "object_keys": [Path(uuid4().hex[:6]).as_posix() for _ in range(3)],
+        },
+    ]
+
+    parameters_doc_object = [
+        {
+            "parameterName": "param_name_1",
+            "parameterValues": [
+                {
+                    "parameterValueId": 1,
+                    "parameterValue": json.dumps([[1, 2, 3], [4, 5, 6], [7, 8, 9]]),
+                    "parameterValueStartDate": "2000-01-01",
+                }
+            ],
+        },
+        {
+            "parameterName": "param_name_2",
+            "parameterValues": [
+                {
+                    "parameterValueId": 2,
+                    "parameterValue": json.dumps(
+                        {
+                            "__file__": {
+                                "bucket": "data",
+                                "objectKey": f"parameters/param_name/{uuid4().hex}.dat",
+                            }
+                        }
+                    ),
+                    "parameterValueStartDate": "2000-01-01",
+                },
+                {
+                    "parameterValueId": 3,
+                    "parameterValue": json.dumps(
+                        {
+                            "__file__": {
+                                "bucket": "data",
+                                "objectKey": f"parameters/param_name/{uuid4().hex}.dat",
+                            }
+                        }
+                    ),
+                    "parameterValueStartDate": "2000-01-02",
+                },
+            ],
+        },
+        {
+            "parameterName": "param_name_4",
+            "parameterValues": [
+                {
+                    "parameterValueId": 4,
+                    "parameterValue": json.dumps(
+                        {"a": 1, "b": 3.14159, "c": "foo", "d": [1, 2, 3]}
+                    ),
+                    "parameterValueStartDate": "2000-01-01",
+                }
+            ],
+        },
+    ]
+
     def __init__(self, *args, **kwargs):
         pass
 
@@ -352,7 +425,9 @@ class FakeGQLClient:
                                     InputDatasetInputDatasetPartResponse(
                                         inputDatasetPart=InputDatasetPartResponse(
                                             inputDatasetPartId=1,
-                                            inputDatasetPartDocument='[{"parameterName": "", "parameterValues": [{"parameterValueId": 1, "parameterValue": "[[1,2,3],[4,5,6],[7,8,9]]", "parameterValueStartDate": "1/1/2000"}]}]',
+                                            inputDatasetPartDocument=json.dumps(
+                                                self.parameters_doc_object
+                                            ),
                                             inputDatasetPartType=InputDatasetPartTypeResponse(
                                                 inputDatasetPartTypeName="parameters"
                                             ),
@@ -361,15 +436,9 @@ class FakeGQLClient:
                                     InputDatasetInputDatasetPartResponse(
                                         inputDatasetPart=InputDatasetPartResponse(
                                             inputDatasetPartId=2,
-                                            inputDatasetPartDocument="""[
-                                            {
-                                                "bucket": "bucket_name",
-                                                "object_keys": [
-                                                    "key1",
-                                                    "key2"
-                                                ]
-                                            }
-                                        ]""",
+                                            inputDatasetPartDocument=json.dumps(
+                                                self.observe_frames_doc_object
+                                            ),
                                             inputDatasetPartType=InputDatasetPartTypeResponse(
                                                 inputDatasetPartTypeName="observe_frames"
                                             ),
@@ -378,15 +447,9 @@ class FakeGQLClient:
                                     InputDatasetInputDatasetPartResponse(
                                         inputDatasetPart=InputDatasetPartResponse(
                                             inputDatasetPartId=3,
-                                            inputDatasetPartDocument="""[
-                                            {
-                                                "bucket": "bucket_name",
-                                                "object_keys": [
-                                                    "key3",
-                                                    "key4"
-                                                ]
-                                            }
-                                        ]""",
+                                            inputDatasetPartDocument=json.dumps(
+                                                self.calibration_frames_doc_object
+                                            ),
                                             inputDatasetPartType=InputDatasetPartTypeResponse(
                                                 inputDatasetPartTypeName="calibration_frames"
                                             ),
@@ -417,14 +480,6 @@ class FakeGQLClient:
         ...
 
 
-class FakeGQLClientNoRecipeConfiguration(FakeGQLClient):
-    def execute_gql_query(self, **kwargs):
-        response = super().execute_gql_query(**kwargs)
-        if type(response[0]) == RecipeRunResponse:
-            response[0].configuration = {}
-        return response
-
-
 # All the following stuff is copied from dkist-processing-pac
 def compute_telgeom(time_hst: Time):
     dkist_lon = (156 + 15 / 60.0 + 21.7 / 3600.0) * (-1)
@@ -774,43 +829,21 @@ def task_with_input_dataset(
         yield task
 
 
-def create_parameter_files(task: WorkflowTaskBase, expected_parameters: dict):
+def create_parameter_files(
+    task: WorkflowTaskBase, parameters_doc: list[dict] = FakeGQLClient.parameters_doc_object
+):
     """
-    Create the parameter files required by the task.
-
-    Parameters
-    ----------
-    task
-        The task associated with these parameters
-
-    expected_parameters
-        A dict of parameters with the format shown below
-
-    Returns
-    -------
-    None
-
-    expected_parameters is a dict with the parameter names as the keys
-    and the values are a list of value dicts for each parameter:
-    expected_parameters =
-        { 'parameter_name_1': [param_dict_1, param_dict_2, ...],
-          'parameter_name_2': [param_dict_1, param_dict_2, ...],
-           ...
-        }
-    where the param_dicts have the following format:
-    sample_param_dict =
-        { "parameterValueId": <param_id>,
-          "parameterValue": <param_value>,
-          "parameterValueStartDate": <start_date>
-        }
+    Create the parameter files specified in the parameters document returned by the metadata store.
+
+    This fixture assumes that the JSON parameters document has already been loaded into a python
+    structure, but the parameter values themselves are still JSON.
     """
-    # Loop over all the parameter values. Each value is a list of parameterValue dicts
-    for expected_parameter_values in expected_parameters.values():
-        for value_dict in expected_parameter_values:
-            if "__file__" not in value_dict["parameterValue"]:
+    for parameter in parameters_doc:
+        for value in parameter["parameterValues"]:
+            if "__file__" not in value["parameterValue"]:
                 continue
-            value = json.loads(value_dict["parameterValue"])
-            param_path = value["__file__"]["objectKey"]
+            parameter_value = json.loads(value["parameterValue"])
+            param_path = parameter_value["__file__"]["objectKey"]
             file_path = task.scratch.workflow_base_path / Path(param_path)
             if not file_path.parent.exists():
                 file_path.parent.mkdir(parents=True, exist_ok=True)

{dkist_processing_common-10.6.1rc2 → dkist_processing_common-10.6.1rc4}/dkist_processing_common/tests/test_input_dataset.py

@@ -308,26 +308,8 @@ def test_input_dataset_parameters(
     task = task_with_input_dataset
     doc_part, _ = input_dataset_parts
     doc_part = doc_part or []  # None case parsing of expected values
-    """
-    expected_parameters is a dict with the parameter names as the keys
-    and the values are a list of value dicts for each parameter:
-    expected_parameters =
-        { 'parameter_name_1': [param_dict_1, param_dict_2, ...],
-          'parameter_name_2': [param_dict_1, param_dict_2, ...],
-           ...
-        }
-    where the param_dicts have the following format:
-    sample_param_dict =
-        { "parameterValueId": <param_id>,
-          "parameterValue": <param_value>,
-          "parameterValueStartDate": <start_date>
-        }
-    """
-    expected_parameters = dict()
-    for item in doc_part:
-        expected_parameters[item["parameterName"]] = item["parameterValues"]
-    create_parameter_files(task, expected_parameters)
-    # key is param name, values is list of InputDatasetParameterValue objects
+    create_parameter_files(task, doc_part)
+    expected_parameters = {item["parameterName"]: item["parameterValues"] for item in doc_part}
     for key, values in task.input_dataset_parameters.items():
         assert key in expected_parameters
         expected_values = expected_parameters[key]