dkist-processing-common 10.5.4__py3-none-any.whl → 12.1.0rc1__py3-none-any.whl

dkist_processing_common/tests/test_transfer_l1_output_data.py

@@ -5,7 +5,6 @@ import pytest
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks import TransferL1Data
-from dkist_processing_common.tests.conftest import FakeGQLClient
 
 
 def fake_list_objects(self, bucket, prefix=None):
@@ -37,7 +36,7 @@ def transfer_l1_data_task(recipe_run_id, tmp_path, fake_constants_db):
         task._purge()
 
 
-def test_transfer_l1_data(transfer_l1_data_task, mocker):
+def test_transfer_l1_data(transfer_l1_data_task, mocker, fake_gql_client):
     """
     Given: A task with frames and movies tagged as output
     When: Transfering the L1 data
@@ -45,7 +44,7 @@ def test_transfer_l1_data(transfer_l1_data_task, mocker):
     """
     # Yeah, we mock a whole bunch of stuff here, but this test at least confirms that the setup to these calls is correct
     mocker.patch(
-        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
     )
     mocker.patch(
         "dkist_processing_common.tasks.mixin.globus.GlobusMixin.globus_transfer_scratch_to_object_store"

dkist_processing_common/tests/test_trial_catalog.py

@@ -1,4 +1,5 @@
 """Tests for the tasks.trial_catalog module."""
+
 from pathlib import Path
 from string import ascii_uppercase
 from uuid import uuid4
@@ -7,19 +8,34 @@ import astropy.units as u
 import pytest
 from astropy.io import fits
 from dkist_data_simulator.spec214.vbi import SimpleVBIDataset
-from hashids import Hashids
+from sqids import Sqids
 
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.asdf import asdf_decoder
+from dkist_processing_common.codecs.basemodel import basemodel_encoder
 from dkist_processing_common.codecs.bytes import bytes_decoder
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
 from dkist_processing_common.codecs.json import json_decoder
 from dkist_processing_common.codecs.quality import quality_data_encoder
+from dkist_processing_common.models.input_dataset import InputDatasetParameter
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks import CreateTrialAsdf
 from dkist_processing_common.tasks import CreateTrialDatasetInventory
 from dkist_processing_common.tasks import CreateTrialQualityReport
-from dkist_processing_common.tests.conftest import FakeGQLClientNoRecipeConfiguration
+from dkist_processing_common.tests.mock_metadata_store import input_dataset_parameters_part_factory
+
+
+@pytest.fixture()
+def mock_input_dataset_parts() -> InputDatasetPartDocumentList:
+    """An InputDatasetPartDocumentList with two parameters, each with one value and a date."""
+    raw = input_dataset_parameters_part_factory(
+        parameter_count=2,
+        parameter_value_count=1,
+        has_date=True,
+        has_file=False,
+    )
+    return InputDatasetPartDocumentList.model_validate({"doc_list": raw})
 
 
 @pytest.fixture()
@@ -41,17 +57,40 @@ def scratch_with_l1_frames(recipe_run_id, tmp_path) -> WorkflowFileSystem:
         scratch.write(
             file_obj, tags=[Tag.output(), Tag.frame()], relative_path=f"{uuid4().hex}.dat"
         )
+
+    return scratch
+
+
+@pytest.fixture()
+def scratch_with_l1_frames_and_parameters(
+    scratch_with_l1_frames, mock_input_dataset_parts
+) -> WorkflowFileSystem:
+    """Scratch instance for a recipe run id with tagged L1 frames and input parameters."""
+    scratch = scratch_with_l1_frames
+
+    # Write validated Pydantic model bytes expected by InputDatasetPartDocumentList
+    file_obj = basemodel_encoder(mock_input_dataset_parts)
+    scratch.write(
+        file_obj,
+        tags=Tag.input_dataset_parameters(),
+        relative_path=f"{uuid4().hex}.json",
+    )
     return scratch
 
 
 @pytest.fixture()
 def create_trial_dataset_inventory_task(
-    recipe_run_id, tmp_path, scratch_with_l1_frames, fake_constants_db, mocker
+    recipe_run_id,
+    tmp_path,
+    scratch_with_l1_frames,
+    fake_constants_db,
+    mocker,
+    fake_gql_client,
 ) -> CreateTrialDatasetInventory:
     """An instance of CreateTrialDatasetInventory with L1 frames tagged in scratch."""
     mocker.patch(
         "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
-        new=FakeGQLClientNoRecipeConfiguration,
+        new=fake_gql_client,
     )
     task = CreateTrialDatasetInventory(
         recipe_run_id=recipe_run_id,
@@ -80,6 +119,22 @@ def create_trial_asdf_task(
     task._purge()
 
 
+@pytest.fixture(scope="function")
+def create_trial_asdf_task_with_params(
+    recipe_run_id, tmp_path, scratch_with_l1_frames_and_parameters, fake_constants_db
+) -> CreateTrialAsdf:
+    """An instance of CreateTrialAsdf with L1 frames and input parameters tagged in scratch."""
+    task = CreateTrialAsdf(
+        recipe_run_id=recipe_run_id,
+        workflow_name="trial_asdf",
+        workflow_version="trial_asdf_version",
+    )
+    task.scratch = scratch_with_l1_frames_and_parameters
+    task.constants._update(fake_constants_db)
+    yield task
+    task._purge()
+
+
 @pytest.fixture()
 def create_trial_quality_report_task(
     recipe_run_id, tmp_path, fake_constants_db
@@ -138,25 +193,32 @@ def test_create_trial_dataset_inventory(create_trial_dataset_inventory_task):
     assert len(inventory) > 20  # a bunch
 
 
-def test_create_trial_asdf(create_trial_asdf_task, recipe_run_id):
+@pytest.mark.parametrize("with_params", [False, True], ids=["no_params", "with_params"])
+def test_create_trial_asdf(with_params, request, recipe_run_id, mock_input_dataset_parts):
     """
     :Given: An instance of CreateTrialAsdf with L1 frames tagged in scratch
     :When: CreateTrialAsdf is run
     :Then: An asdf file for the dataset is tagged in scratch
     """
-    task = create_trial_asdf_task
+    task = request.getfixturevalue(
+        "create_trial_asdf_task_with_params" if with_params else "create_trial_asdf_task"
+    )
     # When
     task()
+
     # Then
     asdf_tags = [Tag.output(), Tag.asdf()]
     filepaths = list(task.scratch.find_all(tags=asdf_tags))
     assert len(filepaths) == 1
-    dataset_id = Hashids(min_length=5, alphabet=ascii_uppercase).encode(recipe_run_id)
-    assert filepaths[0].name == f"INSTRUMENT_L1_20240416T160000_{dataset_id}_user_tools.asdf"
+    dataset_id = Sqids(min_length=6, alphabet=ascii_uppercase).encode([recipe_run_id])
+    assert filepaths[0].name == f"INSTRUMENT_L1_20240416T160000_{dataset_id}_metadata.asdf"
+
     results = list(task.read(tags=asdf_tags, decoder=asdf_decoder))
     assert len(results) == 1
+
     tree = results[0]
     assert isinstance(tree, dict)
+
     for file_name in tree["dataset"].files.filenames:
         # This is a slightly better than check that `not Path(file_name).is_absolute()` because it confirms
         # we've correctly stripped the path of *all* parents (not just those that start at root).
@@ -164,6 +226,19 @@ def test_create_trial_asdf(create_trial_asdf_task, recipe_run_id):
         # `scratch.workflow_base_path`
         assert Path(file_name).name == file_name
 
+    # Only check parameters when present
+    ds = tree["dataset"]
+    assert "parameters" in ds.meta
+    parameters = ds.meta["parameters"]
+    assert isinstance(parameters, list)
+    if with_params:
+        assert parameters, f"ASDF tree must include input parameters: {parameters}"
+        assert len(parameters) == len(mock_input_dataset_parts.doc_list)
+        for param in parameters:
+            assert InputDatasetParameter.model_validate(param) in mock_input_dataset_parts.doc_list
+    else:
+        assert ds.meta["parameters"] == []
+
 
 def test_create_trial_quality_report(create_trial_quality_report_task):
     """

dkist_processing_common/tests/test_trial_output_data.py

@@ -6,11 +6,13 @@ import pytest
 from pydantic.dataclasses import dataclass as validating_dataclass
 
 from dkist_processing_common._util.scratch import WorkflowFileSystem
-from dkist_processing_common.models.graphql import RecipeRunResponse
+from dkist_processing_common.models.graphql import RecipeRunConfiguration
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.mixin.globus import GlobusTransferItem
 from dkist_processing_common.tasks.trial_output_data import TransferTrialData
-from dkist_processing_common.tests.conftest import FakeGQLClient
+from dkist_processing_common.tests.mock_metadata_store import RecipeRunResponseMapping
+from dkist_processing_common.tests.mock_metadata_store import fake_gql_client_factory
+from dkist_processing_common.tests.mock_metadata_store import make_default_recipe_run_response
 
 
 @pytest.fixture
@@ -19,48 +21,46 @@ def destination_bucket() -> str:
 
 
 @pytest.fixture
-def recipe_run_configuration(
+def fake_gql_client_recipe_run_configuration(
     custom_root_name,
     custom_dir_name,
     destination_bucket,
 ):
-    class GQLClientWithConfiguration(FakeGQLClient):
-        def execute_gql_query(self, **kwargs):
-            response = super().execute_gql_query(**kwargs)
-            if isinstance(response, list):
-                if isinstance(response[0], RecipeRunResponse):
-                    response[0].configuration = json.dumps(
-                        {
-                            "trial_root_directory_name": custom_root_name,
-                            "trial_directory_name": custom_dir_name,
-                            "destination_bucket": destination_bucket,
-                        }
-                    )
-            return response
-
-    return GQLClientWithConfiguration
+    recipe_run_response = make_default_recipe_run_response()
+    configuration = RecipeRunConfiguration(
+        trial_root_directory_name=custom_root_name,
+        trial_directory_name=custom_dir_name,
+        destination_bucket=destination_bucket,
+    )
+    recipe_run_response.configuration = configuration.model_dump_json()
+
+    new_response_mapping = RecipeRunResponseMapping(response=recipe_run_response)
+    FakeGQLClientWithConfiguration = fake_gql_client_factory(
+        response_mapping_override=new_response_mapping
+    )
+
+    return FakeGQLClientWithConfiguration
 
 
 @pytest.fixture
-def recipe_run_configuration_with_tag_lists(
+def fake_gql_client_recipe_run_configuration_with_tag_lists(
     custom_root_name, custom_dir_name, destination_bucket, exclusive_tag_lists
 ):
-    class GQLClientWithConfiguration(FakeGQLClient):
-        def execute_gql_query(self, **kwargs):
-            response = super().execute_gql_query(**kwargs)
-            if isinstance(response, list):
-                if isinstance(response[0], RecipeRunResponse):
-                    response[0].configuration = json.dumps(
-                        {
-                            "trial_root_directory_name": custom_root_name,
-                            "trial_directory_name": custom_dir_name,
-                            "destination_bucket": destination_bucket,
-                            "trial_exclusive_transfer_tag_lists": exclusive_tag_lists,
-                        }
-                    )
-            return response
-
-    return GQLClientWithConfiguration
+    recipe_run_response = make_default_recipe_run_response()
+    configuration = RecipeRunConfiguration(
+        trial_root_directory_name=custom_root_name,
+        trial_directory_name=custom_dir_name,
+        destination_bucket=destination_bucket,
+        trial_exclusive_transfer_tag_lists=exclusive_tag_lists,
+    )
+    recipe_run_response.configuration = configuration.model_dump_json()
+
+    new_response_mapping = RecipeRunResponseMapping(response=recipe_run_response)
+    FakeGQLClientWithConfiguration = fake_gql_client_factory(
+        response_mapping_override=new_response_mapping
+    )
+
+    return FakeGQLClientWithConfiguration
 
 
 @pytest.fixture
@@ -70,11 +70,11 @@ def trial_output_task() -> type[TransferTrialData]:
 
 @pytest.fixture
 def basic_trial_output_task(
-    recipe_run_id, recipe_run_configuration, trial_output_task, tmp_path, mocker
+    recipe_run_id, fake_gql_client_recipe_run_configuration, trial_output_task, tmp_path, mocker
 ):
     mocker.patch(
         "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
-        new=recipe_run_configuration,
+        new=fake_gql_client_recipe_run_configuration,
     )
     proposal_id = "test_proposal_id"
     with trial_output_task(
@@ -121,10 +121,10 @@ class OutputFileNames:
 def complete_trial_output_task(
     request, recipe_run_id, trial_output_task, tmp_path, mocker
 ) -> tuple[TransferTrialData, str, OutputFileObjects]:
-    recipe_run = request.param
+    fake_gql_client = request.param
     mocker.patch(
         "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
-        new=request.getfixturevalue(recipe_run),
+        new=request.getfixturevalue(fake_gql_client),
     )
     proposal_id = "test_proposal_id"
     with trial_output_task(
@@ -158,13 +158,12 @@ def complete_trial_output_task(
         task.write(asdf_file_obj, relative_path=asdf_file_name, tags=[Tag.output(), Tag.asdf()])
 
         # Write quality data
-        # quality data is not tagged as OUTPUT
         quality_data_obj = uuid4().hex.encode("utf8")
         quality_data_name = "quality_data.json"
         task.write(
             quality_data_obj,
             relative_path=quality_data_name,
-            tags=Tag.quality_data(),
+            tags=[Tag.output(), Tag.quality_data()],
         )
 
         # Write a quality report file
@@ -241,7 +240,9 @@ def test_format_object_key(
     )
 
 
-@pytest.mark.parametrize("complete_trial_output_task", ["recipe_run_configuration"], indirect=True)
+@pytest.mark.parametrize(
+    "complete_trial_output_task", ["fake_gql_client_recipe_run_configuration"], indirect=True
+)
 @pytest.mark.parametrize(
     "custom_root_name, custom_dir_name",
     [
@@ -289,7 +290,9 @@ def test_build_transfer_list(
 
 
 @pytest.mark.parametrize(
-    "complete_trial_output_task", ["recipe_run_configuration_with_tag_lists"], indirect=True
+    "complete_trial_output_task",
+    ["fake_gql_client_recipe_run_configuration_with_tag_lists"],
+    indirect=True,
 )
 @pytest.mark.parametrize(
     "custom_root_name, custom_dir_name, exclusive_tag_lists, expected_output",
@@ -332,33 +335,3 @@ def test_build_transfer_list_with_exclusive_tag_lists(
     assert transfer_item.destination_path == expected_destination_path
     with transfer_item.source_path.open(mode="rb") as f:
         assert file_obj == f.read()
-
-
-@pytest.mark.parametrize(
-    "complete_trial_output_task", ["recipe_run_configuration_with_tag_lists"], indirect=True
-)
-@pytest.mark.parametrize(
-    "custom_root_name, custom_dir_name, exclusive_tag_lists",
-    [
-        pytest.param(
-            None, None, [Tag.task("TASKY_MCTASKERSON")], id="Default trial dir and trial root names"
-        )
-    ],
-)
-def test_build_transfer_list_with_bad_exclusive_tag_lists(
-    complete_trial_output_task,
-    destination_bucket,
-    custom_dir_name,
-    custom_root_name,
-    exclusive_tag_lists,
-):
-    """
-    :Given: A Task based on TrialTransferDataBase
-    :When: The exclusive tag list is a simple list instead of the required list of lists
-    :Then: The correct value error is raised
-    """
-    task, proposal_id, output_file_objects, output_file_names = complete_trial_output_task
-
-    with pytest.raises(ValueError) as ve:
-        transfer_list = task.build_transfer_list()
-    assert f"tag_lists={exclusive_tag_lists} must" in str(ve)

dkist_processing_common/tests/test_workflow_task_base.py

@@ -2,13 +2,12 @@ import json
 import logging
 import re
 import tomllib
-from dataclasses import asdict
+from importlib.metadata import version
 from pathlib import Path
 from string import ascii_uppercase
 
 import pytest
-from hashids import Hashids
-from pkg_resources import get_distribution
+from sqids import Sqids
 
 import dkist_processing_common
 from dkist_processing_common._util.scratch import WorkflowFileSystem
@@ -183,16 +182,15 @@ def test_dataset_id(workflow_data_task):
     Then: the dataset id hashed from the recipe run id is returned
     """
     task = workflow_data_task[0]
-    assert task.constants.dataset_id == Hashids(min_length=5, alphabet=ascii_uppercase).encode(
-        task.recipe_run_id
-    )
+    expected_dataset_id = Sqids(min_length=6, alphabet=ascii_uppercase).encode([task.recipe_run_id])
+    assert len(expected_dataset_id) >= 6
+    assert task.constants.dataset_id == expected_dataset_id
 
 
 class ProvenanceTask(WorkflowTaskBase):
     record_provenance = True
 
-    def run(self):
-        ...
+    def run(self): ...
 
     # Because I couldn't figure out how to mock the mixin
     def metadata_store_record_provenance(self, is_task_manual: bool, library_versions: str):
@@ -204,7 +202,7 @@ class ProvenanceTask(WorkflowTaskBase):
             libraryVersions=library_versions,
             workflowVersion=self.workflow_version,
         )
-        self.write(data=bytes(json.dumps(asdict(params)), "utf-8"), tags=["TEST_PROVENANCE"])
+        self.write(data=bytes(params.model_dump_json(), "utf-8"), tags=["TEST_PROVENANCE"])
 
 
 @pytest.fixture(scope="function")
@@ -272,7 +270,7 @@ def test_library_versions(provenance_task, package_dependencies):
     #   installed packages.
     for package in package_dependencies:
         assert package in libraries
-        assert libraries[package] == get_distribution(package).version
+        assert libraries[package] == version(package)
 
 
 def test_record_provenance(provenance_task):