dkist-processing-test 1.21.3rc1__py3-none-any.whl → 1.23.8rc1__py3-none-any.whl

dkist_processing_test/models/parameters.py

@@ -1,3 +1,4 @@
+import json
 from datetime import datetime
 
 import numpy as np
@@ -33,3 +34,18 @@ class TestParameters(ParameterBase, ParameterWavelengthMixin):
     def wavelength_category(self) -> str:
         """A dummy parameter that depends on wavelength."""
         return self._find_parameter_closest_wavelength("test_wavelength_category")
+
+    @property
+    def value_message(self) -> str:
+        """A dummy parameter that returns a message."""
+        return self._find_most_recent_past_value("test_message")
+
+    @property
+    def file_message(self) -> str:
+        """A dummy parameter that returns a message from a file."""
+        param_dict = self._find_most_recent_past_value("test_message_file")
+        file_path = param_dict["param_path"]
+        with open(file_path, "r") as f:
+            message = json.load(f)
+
+        return message

dkist_processing_test/tasks/__init__.py

@@ -6,5 +6,4 @@ from dkist_processing_test.tasks.movie import *
 from dkist_processing_test.tasks.noop import *
 from dkist_processing_test.tasks.parse import *
 from dkist_processing_test.tasks.quality import *
-from dkist_processing_test.tasks.trial_output_data import *
 from dkist_processing_test.tasks.write_l1 import *

dkist_processing_test/tasks/fake_science.py

@@ -10,6 +10,7 @@ from dkist_processing_common.codecs.json import json_encoder
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks import WorkflowTaskBase
 from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetMixin
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_test.models.parameters import TestParameters
 
@@ -57,10 +58,15 @@ class GenerateCalibratedData(WorkflowTaskBase, InputDatasetMixin):
 
             self.write(data={"test": "dictionary"}, tags=["BAZ"], encoder=json_encoder)
 
-        with self.apm_task_step("Creating frames that won't be used"):
+        with self.apm_task_step(
+            "Creating frames that won't be used or transferred as trial outputs"
+        ):
             self.write(data=b"123", tags=[Tag.intermediate(), Tag.task("NOT_USED"), Tag.frame()])
             self.write(data=b"123", tags=["FOO"])
 
+        logger.info(f"Using {self.parameters.value_message = }")
+        logger.info(f"Using {self.parameters.file_message = }")
+
         with self.apm_task_step("Loop over inputs"):
             count = 1  # keep a running count to increment the dsps repeat number
             for hdu in self.read(tags=Tag.input(), decoder=fits_hdu_decoder):
@@ -81,6 +87,10 @@ class GenerateCalibratedData(WorkflowTaskBase, InputDatasetMixin):
                     header["VBINMOSC"] = self.constants.num_dsps_repeats
                     header["VBICMOSC"] = count
 
+                    # Sneak date-dependent parameter values into header for end-to-end checking
+                    header["CAM_ID"] = self.parameters.file_message
+                    header["CAMERA"] = self.parameters.value_message
+
                     output_hdu = fits.PrimaryHDU(data=data, header=header)
 
                     wavelength_category = self.parameters.wavelength_category

dkist_processing_test/tests/conftest.py

@@ -48,10 +48,20 @@ def generate_214_l0_fits_frame(
 
 
 @pytest.fixture(scope="session")
-def parameter_file_object_key() -> str:
+def array_parameter_file_object_key() -> str:
     return "random.fits"
 
 
+@pytest.fixture(scope="session")
+def early_json_parameter_file_object_key() -> str:
+    return "early_message.json"
+
+
+@pytest.fixture(scope="session")
+def late_json_parameter_file_object_key() -> str:
+    return "late_message.json"
+
+
 @pytest.fixture(scope="session")
 def random_parameter_hdulist() -> (fits.HDUList, float, float, float):
     rng = np.random.default_rng()
@@ -62,3 +72,33 @@ def random_parameter_hdulist() -> (fits.HDUList, float, float, float):
     hdul = fits.HDUList([fits.PrimaryHDU(rand_data), fits.ImageHDU(const_data)])
 
     return hdul, mu, std, const
+
+
+@pytest.fixture(scope="session")
+def early_file_message_str() -> str:
+    return "Early in a file"
+
+
+@pytest.fixture(scope="session")
+def late_file_message_str() -> str:
+    return "Late in a file"
+
+
+@pytest.fixture(scope="session")
+def early_value_message_str() -> str:
+    return "Early"
+
+
+@pytest.fixture(scope="session")
+def late_value_message_str() -> str:
+    return "Late"
+
+
+@pytest.fixture(scope="session")
+def early_date() -> str:
+    return "1980-01-01"
+
+
+@pytest.fixture(scope="session")
+def late_date() -> str:
+    return "2000-01-01"

dkist_processing_test/tests/parameter_models.py

@@ -0,0 +1,130 @@
+import json
+from datetime import datetime
+from random import randint
+from typing import Any
+from typing import Union
+
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import Field
+from pydantic import field_serializer
+from pydantic import field_validator
+from pydantic import model_serializer
+
+param_value_type_hint = Union[
+    int, str, list, tuple, "WavelengthParameterValue", "RawFileParameterValue"
+]
+multi_str_type = Union[str, "MultipleParameterValues"]
+multi_file_type = Union["RawFileParameterValue", "MultipleParameterValues"]
+multi_wave_type = Union["WavelengthParameterValue", "MultipleParameterValues"]
+
+
+class ParameterValue(BaseModel):
+    """A single parameterValue entry"""
+
+    parameterValue: param_value_type_hint
+    parameterValueId: int = Field(default_factory=lambda: randint(1000, 2000))
+    parameterValueStartDate: datetime = datetime(1946, 11, 20)
+
+    @field_serializer("parameterValue")
+    def json_parameter_value(self, parameter_value: param_value_type_hint) -> str:
+        """Encode the actual value in a JSON string."""
+        try:
+            parameter_value = parameter_value.model_dump()
+        except:
+            # If the value is just a basic type (i.e., not a `BaseModel`)
+            pass
+        return json.dumps(parameter_value)
+
+    @field_serializer("parameterValueStartDate")
+    def datetime_iso_format(self, start_datetime: datetime) -> str:
+        """Encode the start date as an ISO-formatted string."""
+        return start_datetime.isoformat()
+
+
+class MultipleParameterValues(BaseModel):
+    """
+    Container for a list of parameterValues.
+
+    This exists to be different than a raw `list`, which is a valid `parameterValue.parameterValue` type
+    """
+
+    parameter_value_list: list[ParameterValue]
+
+    @field_validator("parameter_value_list", mode="before")
+    @classmethod
+    def ensure_list_of_parameter_values(cls, input_list: list) -> list[ParameterValue]:
+        """Convert any raw types to `ParameterValue` objects."""
+        output_list = []
+        for parameter_value in input_list:
+            if not isinstance(parameter_value, ParameterValue):
+                parameter_value = ParameterValue(parameterValue=parameter_value)
+
+            output_list.append(parameter_value)
+
+        return output_list
+
+    @field_validator("parameter_value_list")
+    @classmethod
+    def no_repeat_start_dates(
+        cls, parameter_value_list: list[ParameterValue]
+    ) -> list[ParameterValue]:
+        """Fail validation if any of the `ParameterValues` have the same `parameterValueStartDate."""
+        start_dates = [pv.parameterValueStartDate for pv in parameter_value_list]
+        if len(set(start_dates)) != len(start_dates):
+            raise ValueError(
+                f"parameterValueStartDates must be unique. Got {set(start_dates)} over {len(start_dates)} parameters."
+            )
+
+        return parameter_value_list
+
+    @model_serializer
+    def parameter_value_list(self):
+        """Return just the list of `ParameterValues`."""
+        return self.parameter_value_list
+
+
+class WavelengthParameterValue(BaseModel):
+    values: tuple
+    wavelength: tuple = (1.0, 2.0, 3.0)
+
+
+class RawFileParameterValue(BaseModel):
+    """
+    For parameters that are files on disk.
+
+    "Raw" in the sense that it still has the `__file__`-level dictionary.
+    """
+
+    objectKey: str
+    bucket: str = "doesn't_matter"
+
+    @model_serializer
+    def file_dict(self) -> dict:
+        """Wrap the input values in a `__file__` dict."""
+        return {"__file__": dict(self)}
+
+
+class TestParameterValues(BaseModel):
+    test_random_data: multi_file_type = RawFileParameterValue(objectKey="")
+    test_wavelength_category: multi_wave_type = WavelengthParameterValue(
+        values=("one", "two", "three")
+    )
+    test_message: multi_str_type = "Weird?"
+    test_message_file: multi_file_type = RawFileParameterValue(objectKey="")
+
+    model_config = ConfigDict(validate_default=True)
+
+    @field_validator("*")
+    @classmethod
+    def ensure_parameter_value_lists(cls, parameter: Any) -> MultipleParameterValues:
+        """Convert all values to `MultipleParameterValues`, if they aren't already."""
+        if not isinstance(parameter, MultipleParameterValues):
+            return MultipleParameterValues(parameter_value_list=[parameter])
+
+        return parameter
+
+    @model_serializer
+    def input_dataset_document_parameters_part(self) -> list:
+        """Place all parameters into higher-level dictionaries required by the input dataset document."""
+        return [{"parameterName": f, "parameterValues": v} for f, v in self]

dkist_processing_test/tests/test_parameters.py

@@ -1,3 +1,4 @@
+import json
 from datetime import datetime
 
 import numpy as np
@@ -9,16 +10,16 @@ from dkist_processing_test.models.parameters import TestParameters
 
 
 @pytest.fixture(scope="session")
-def file_parameter(
-    tmp_path_factory, random_parameter_hdulist, parameter_file_object_key
+def array_file_parameter(
+    tmp_path_factory, random_parameter_hdulist, array_parameter_file_object_key
 ) -> tuple[InputDatasetParameterValue, float, float, float]:
     hdul, mu, std, const = random_parameter_hdulist
-    file_path = tmp_path_factory.mktemp("parameters") / parameter_file_object_key
+    file_path = tmp_path_factory.mktemp("parameters") / array_parameter_file_object_key
     hdul.writeto(file_path)
 
     value = {
         "bucket": "raw",
-        "objectKey": parameter_file_object_key,
+        "objectKey": array_parameter_file_object_key,
         "param_path": file_path,
         "is_file": True,
     }
@@ -31,12 +32,48 @@ def file_parameter(
     return parameter_value, mu, std, const
 
 
+@pytest.fixture(scope="session")
+def message_file_parameter(
+    tmp_path_factory, early_file_message_str, early_json_parameter_file_object_key
+):
+
+    file_path = tmp_path_factory.mktemp("parameters") / early_json_parameter_file_object_key
+    with open(file_path, "w") as f:
+        json.dump(early_file_message_str, f)
+
+    value = {
+        "bucket": "raw",
+        "objectKey": early_json_parameter_file_object_key,
+        "param_path": file_path,
+        "is_file": True,
+    }
+
+    parameter_value = InputDatasetParameterValue(
+        parameter_value_id=2,
+        parameter_value=value,
+        parameter_value_start_date=datetime(1946, 11, 20),
+    )
+    return parameter_value
+
+
+@pytest.fixture(scope="session")
+def message_parameter(early_value_message_str):
+
+    parameter_value = InputDatasetParameterValue(
+        parameter_value_id=3,
+        parameter_value=early_value_message_str,
+        parameter_value_start_date=datetime(1946, 11, 20),
+    )
+
+    return parameter_value
+
+
 @pytest.fixture(scope="session")
 def wavelength_parameter() -> InputDatasetParameterValue:
     value = {"wavelength": [1.0, 2.0, 3.0], "values": ["one", "two", "three"]}
 
     parameter_value = InputDatasetParameterValue(
-        parameter_value_id=2,
+        parameter_value_id=4,
         parameter_value=value,
         parameter_value_start_date=datetime(1946, 11, 20),
     )
@@ -44,12 +81,16 @@ def wavelength_parameter() -> InputDatasetParameterValue:
 
 
 @pytest.fixture(scope="session")
-def parameter_dict_with_path(file_parameter, wavelength_parameter):
+def parameter_dict_with_path(
+    array_file_parameter, message_file_parameter, message_parameter, wavelength_parameter
+):
     """Enough of an input dataset parameters part to exercise file loading parameters."""
-    file_parameter_value, _, _, _ = file_parameter
+    file_parameter_value, _, _, _ = array_file_parameter
 
     param_dict = {
         "test_random_data": [file_parameter_value],
+        "test_message": [message_parameter],
+        "test_message_file": [message_file_parameter],
         "test_wavelength_category": [wavelength_parameter],
     }
 
@@ -87,17 +128,24 @@ def task_with_parameters(parameter_dict_with_path):
     return task
 
 
-def test_parameter(task_with_parameters, file_parameter):
+def test_parameter(
+    task_with_parameters,
+    array_file_parameter,
+    early_file_message_str,
+    early_value_message_str,
+):
     """
     Given: A task with parameters that depend on files
     When: Accessing those parameters
     Then: The correct values are returned
     """
     task = task_with_parameters
-    _, mu, std, const = file_parameter
+    _, mu, std, const = array_file_parameter
 
     assert type(task.parameters.randomness) is tuple
     np.testing.assert_allclose(np.array(task.parameters.randomness), np.array([mu, std]), rtol=1)
 
     assert task.parameters.constant == const
     assert task.parameters.wavelength_category == "two"
+    assert task.parameters.value_message == early_value_message_str
+    assert task.parameters.file_message == early_file_message_str

dkist_processing_test/tests/test_tasks.py

@@ -6,8 +6,8 @@ from dataclasses import asdict
 from dataclasses import dataclass
 from dataclasses import is_dataclass
 from datetime import datetime
+from datetime import timedelta
 from random import randint
-from typing import Type
 from uuid import uuid4
 
 import numpy as np
@@ -18,11 +18,12 @@ from dkist_header_validator import spec122_validator
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.fits import fits_hdu_decoder
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
+from dkist_processing_common.codecs.json import json_encoder
 from dkist_processing_common.models.constants import BudName
 from dkist_processing_common.models.constants import ConstantsBase
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.models.task_name import TaskName
-from dkist_processing_common.tasks import QualityL0Metrics
+from dkist_processing_common.tasks import TransferTrialData
 from dkist_processing_common.tests.conftest import FakeGQLClient
 from dkist_service_configuration.logging import logger
 
@@ -34,16 +35,19 @@ from dkist_processing_test.tasks.fake_science import GenerateCalibratedData
 from dkist_processing_test.tasks.movie import AssembleTestMovie
 from dkist_processing_test.tasks.movie import MakeTestMovieFrames
 from dkist_processing_test.tasks.noop import NoOpTask
-from dkist_processing_test.tasks.trial_output_data import TransferTestTrialData
 from dkist_processing_test.tasks.write_l1 import WriteL1Data
 from dkist_processing_test.tests.conftest import generate_214_l0_fits_frame
 from dkist_processing_test.tests.conftest import S122Headers
+from dkist_processing_test.tests.parameter_models import MultipleParameterValues
+from dkist_processing_test.tests.parameter_models import ParameterValue
+from dkist_processing_test.tests.parameter_models import RawFileParameterValue
+from dkist_processing_test.tests.parameter_models import TestParameterValues
 
 
 @dataclass
 class FakeConstantDb:
     NUM_DSPS_REPEATS: int = 2
-    OBS_IP_START_TIME: str = "2024-06-12T12:00:00"
+    OBS_IP_START_TIME: str = "1990-06-12T12:00:00"
     INSTRUMENT: str = "TEST"
     AVERAGE_CADENCE: float = 10.0
     MINIMUM_CADENCE: float = 10.0
@@ -94,12 +98,25 @@ def generate_calibrated_data_task(
     recipe_run_id,
     assign_input_dataset_doc_to_task,
     link_constants_db,
-    parameter_file_object_key,
+    array_parameter_file_object_key,
     random_parameter_hdulist,
+    early_json_parameter_file_object_key,
+    early_file_message_str,
+    late_json_parameter_file_object_key,
+    late_file_message_str,
+    early_or_late,
+    late_date,
 ):
     number_of_frames = 10
+    if early_or_late == "early":
+        obs_ip_start_time_str = (datetime.fromisoformat(late_date) - timedelta(days=30)).isoformat()
+    elif early_or_late == "late":
+        obs_ip_start_time_str = (datetime.fromisoformat(late_date) + timedelta(days=30)).isoformat()
     link_constants_db(
-        recipe_run_id=recipe_run_id, constants_obj=FakeConstantDb(NUM_DSPS_REPEATS=number_of_frames)
+        recipe_run_id=recipe_run_id,
+        constants_obj=FakeConstantDb(
+            NUM_DSPS_REPEATS=number_of_frames, OBS_IP_START_TIME=obs_ip_start_time_str
+        ),
     )
     with GenerateCalibratedData(
         recipe_run_id=recipe_run_id, workflow_name="GenerateCalibratedData", workflow_version="VX.Y"
@@ -125,13 +142,25 @@ def generate_calibrated_data_task(
                 data=hdul, tags=Tag.input(), relative_path=file_name, encoder=fits_hdulist_encoder
             )
 
-        # Write parameter file
+        # Write parameter files
         hdul = random_parameter_hdulist[0]
         task.write(
-            data=hdul, tags=Tag.parameter(parameter_file_object_key), encoder=fits_hdulist_encoder
+            data=hdul,
+            tags=Tag.parameter(array_parameter_file_object_key),
+            encoder=fits_hdulist_encoder,
+        )
+        task.write(
+            data=early_file_message_str,
+            tags=Tag.parameter(early_json_parameter_file_object_key),
+            encoder=json_encoder,
+        )
+        task.write(
+            data=late_file_message_str,
+            tags=Tag.parameter(late_json_parameter_file_object_key),
+            encoder=json_encoder,
         )
 
-        # This needs to be after we've tagged the parameter file
+        # This needs to be after we've written and tagged the parameter files
         assign_input_dataset_doc_to_task(task, obs_ip_start_time=task.constants.obs_ip_start_time)
 
         # result
@@ -142,55 +171,61 @@ def generate_calibrated_data_task(
 
 
 @pytest.fixture(scope="session")
-def input_dataset_document_parameters_part_with_file(parameter_file_object_key):
-    param_name = "test_random_data"
-    value = {
-        "__file__": {"bucket": "foo", "objectKey": parameter_file_object_key},
-    }
-    value_id = randint(1000, 2000)
-    values = [
-        {
-            "parameterValueId": value_id,
-            "parameterValue": json.dumps(value),
-            "parameterValueStartDate": "1946-11-20",
-        }
-    ]
-    parameter = {"parameterName": param_name, "parameterValues": values}
-    parameters_list = [parameter]
+def input_dataset_document_parameters_part_json(
+    array_parameter_file_object_key,
+    early_json_parameter_file_object_key,
+    late_json_parameter_file_object_key,
+    early_value_message_str,
+    late_value_message_str,
+    early_date,
+    late_date,
+):
 
-    return parameters_list
+    message_file_values = MultipleParameterValues(
+        parameter_value_list=[
+            ParameterValue(
+                parameterValue=RawFileParameterValue(
+                    objectKey=early_json_parameter_file_object_key
+                ),
+                parameterValueStartDate=early_date,
+            ),
+            ParameterValue(
+                parameterValue=RawFileParameterValue(objectKey=late_json_parameter_file_object_key),
+                parameterValueStartDate=late_date,
+            ),
+        ]
+    )
+    message_value_values = MultipleParameterValues(
+        parameter_value_list=[
+            ParameterValue(
+                parameterValue=early_value_message_str,
+                parameterValueStartDate=early_date,
+            ),
+            ParameterValue(
+                parameterValue=late_value_message_str,
+                parameterValueStartDate=late_date,
+            ),
+        ]
+    )
+    parameters_obj = TestParameterValues(
+        test_random_data=RawFileParameterValue(objectKey=array_parameter_file_object_key),
+        test_message_file=message_file_values,
+        test_message=message_value_values,
+    )
 
+    part_json_str = parameters_obj.model_dump_json()
 
-@pytest.fixture(scope="session")
-def input_dataset_document_parameters_part_with_wavelength():
-    param_name = "test_wavelength_category"
-    value = {"wavelength": [1.0, 2.0, 3.0], "values": ["one", "two", "three"]}
-    value_id = randint(2000, 3000)
-    values = [
-        {
-            "parameterValueId": value_id,
-            "parameterValue": json.dumps(value),
-            "parameterValueStartDate": "1946-11-20",
-        }
-    ]
-    parameter = {"parameterName": param_name, "parameterValues": values}
-    parameter_list = [parameter]
-    return parameter_list
+    return part_json_str
 
 
 @pytest.fixture(scope="session")
 def assign_input_dataset_doc_to_task(
-    input_dataset_document_parameters_part_with_file,
-    input_dataset_document_parameters_part_with_wavelength,
+    input_dataset_document_parameters_part_json,
 ):
     def update_task(task, obs_ip_start_time=None):
         doc_path = task.scratch.workflow_base_path / "dataset_parameters.json"
-        full_parameters = (
-            input_dataset_document_parameters_part_with_file
-            + input_dataset_document_parameters_part_with_wavelength
-        )
         with open(doc_path, "w") as f:
-            f.write(json.dumps(full_parameters))
+            f.write(input_dataset_document_parameters_part_json)
         task.tag(doc_path, Tag.input_dataset_parameters())
         task.parameters = TestParameters(
             task.input_dataset_parameters, wavelength=2.0, obs_ip_start_time=obs_ip_start_time
@@ -214,7 +249,16 @@ def constants_linker(recipe_run_id: int, constants_obj):
     return
 
 
-def test_generate_calibrated_data(generate_calibrated_data_task, mocker):
+@pytest.mark.parametrize("early_or_late", [pytest.param("early"), pytest.param("late")])
+def test_generate_calibrated_data(
+    generate_calibrated_data_task,
+    early_file_message_str,
+    late_file_message_str,
+    early_value_message_str,
+    late_value_message_str,
+    early_or_late,
+    mocker,
+):
     """
     Given: A GenerateCalibratedData task
     When: Calling the task instance
@@ -230,12 +274,23 @@ def test_generate_calibrated_data(generate_calibrated_data_task, mocker):
         task.read(tags=[Tag.calibrated(), Tag.frame()], decoder=fits_hdu_decoder)
     )
 
+    if early_or_late == "early":
+        expected_file_message = early_file_message_str
+        expected_value_message = early_value_message_str
+    elif early_or_late == "late":
+        expected_file_message = late_file_message_str
+        expected_value_message = late_value_message_str
+
     # Verify frames
     assert len(calibrated_frame_hdus) == number_of_frames
     for hdu in calibrated_frame_hdus:
         assert "VBINMOSC" in hdu.header
         assert "VBICMOSC" in hdu.header
 
+        # Verify correct date params were used
+        assert hdu.header["CAM_ID"] == expected_file_message
+        assert hdu.header["CAMERA"] == expected_value_message
+
     # Verify debug frame was written
     debug_frame_paths = list(task.read(tags=[Tag.debug(), Tag.frame()]))
     assert len(debug_frame_paths) == 1
@@ -606,7 +661,7 @@ def trial_output_task(recipe_run_id, tmp_path, mocker):
         new=FakeGQLClient,
     )
     proposal_id = "test_proposal_id"
-    with TransferTestTrialData(
+    with TransferTrialData(
         recipe_run_id=recipe_run_id,
         workflow_name="workflow_name",
         workflow_version="workflow_version",
@@ -623,21 +678,14 @@ def trial_output_task(recipe_run_id, tmp_path, mocker):
         task.write(debug_file_obj, tags=[Tag.debug(), Tag.frame()])
         file_count += 1
 
-        # Write an intermediate frame that we want to transfer
-        intermediate_keep_file_obj = uuid4().hex.encode("utf8")
+        # Write an intermediate frame
+        intermediate_file_obj = uuid4().hex.encode("utf8")
         task.write(
-            intermediate_keep_file_obj,
+            intermediate_file_obj,
             tags=[Tag.intermediate(), Tag.frame(), Tag.task("DUMMY")],
         )
         file_count += 1
 
-        # Write an intermediate frame that we don't want to transfer
-        intermediate_discard_file_obj = uuid4().hex.encode("utf8")
-        task.write(
-            intermediate_discard_file_obj,
-            tags=[Tag.intermediate(), Tag.frame(), Tag.task("WHO_CARES")],
-        )
-
         # An output frame
         output_file_obj = uuid4().hex.encode("utf8")
         task.write(output_file_obj, tags=[Tag.output(), Tag.frame()])
@@ -668,12 +716,6 @@ def trial_output_task(recipe_run_id, tmp_path, mocker):
         task.write(quality_report_file_obj, tags=[Tag.output(), Tag.quality_report()])
         file_count += 1
 
-        # Specifically tagged files
-        task.write(uuid4().hex.encode("utf8"), tags=[Tag.frame(), "FOO", "BAR"])
-        file_count += 1
-        task.write(uuid4().hex.encode("utf8"), tags=[Tag.frame(), "BAZ"])
-        file_count += 1
-
         # This one won't get transferred
         task.write(uuid4().hex.encode("utf8"), tags=[Tag.frame(), "FOO"])
 
@@ -683,7 +725,7 @@ def trial_output_task(recipe_run_id, tmp_path, mocker):
 
 def test_transfer_test_trial_data(trial_output_task, mocker):
     """
-    Given: A TransferTestTrialData task with associated frames
+    Given: A TransferTrialData task with associated frames
     When: Running the task and building the transfer list
     Then: No errors occur and the transfer list has the correct number of items
     """
@@ -693,7 +735,7 @@ def test_transfer_test_trial_data(trial_output_task, mocker):
         "dkist_processing_common.tasks.mixin.globus.GlobusMixin.globus_transfer_scratch_to_object_store"
     )
     mocker.patch(
-        "dkist_processing_test.tasks.trial_output_data.TransferTestTrialData.remove_folder_objects"
+        "dkist_processing_common.tasks.trial_output_data.TransferTrialData.remove_folder_objects"
     )
 
     # Just make sure the thing runs with no errors

dkist_processing_test/workflows/common_tasks.py

@@ -10,6 +10,7 @@ from dkist_processing_common.tasks import SubmitDatasetMetadata
 from dkist_processing_common.tasks import Teardown
 from dkist_processing_common.tasks import TransferL0Data
 from dkist_processing_common.tasks import TransferL1Data
+from dkist_processing_common.tasks import TransferTrialData
 from dkist_processing_common.tasks import TrialTeardown
 from dkist_processing_core import Workflow
 
@@ -19,7 +20,6 @@ from dkist_processing_test.tasks import MakeTestMovieFrames
 from dkist_processing_test.tasks import ParseL0TestInputData
 from dkist_processing_test.tasks import TestAssembleQualityData
 from dkist_processing_test.tasks import TestQualityL0Metrics
-from dkist_processing_test.tasks import TransferTestTrialData
 from dkist_processing_test.tasks import WriteL1Data
 from dkist_processing_test.tasks.manual import ManualWithoutProvenance
 from dkist_processing_test.tasks.manual import ManualWithProvenance
@@ -132,7 +132,7 @@ transfer_trial_data = Workflow(
     detail="transfer-trial-data",
     workflow_package=__package__,
 )
-transfer_trial_data.add_node(task=TransferTestTrialData, upstreams=None)
+transfer_trial_data.add_node(task=TransferTrialData, upstreams=None)
 
 # SubmitDatasetMetadata Task
 submit_dataset_metadata = Workflow(

dkist_processing_test/workflows/trial_end_to_end.py

@@ -6,6 +6,7 @@ from dkist_processing_common.tasks import CreateTrialDatasetInventory
 from dkist_processing_common.tasks import CreateTrialQualityReport
 from dkist_processing_common.tasks import QualityL1Metrics
 from dkist_processing_common.tasks import TransferL0Data
+from dkist_processing_common.tasks import TransferTrialData
 from dkist_processing_common.tasks import TrialTeardown
 from dkist_processing_core import Workflow
 
@@ -15,7 +16,6 @@ from dkist_processing_test.tasks import MakeTestMovieFrames
 from dkist_processing_test.tasks import ParseL0TestInputData
 from dkist_processing_test.tasks import TestAssembleQualityData
 from dkist_processing_test.tasks import TestQualityL0Metrics
-from dkist_processing_test.tasks import TransferTestTrialData
 from dkist_processing_test.tasks import WriteL1Data
 
 trial = Workflow(
@@ -51,7 +51,7 @@ trial.add_node(
 
 # Output flow
 trial.add_node(
-    task=TransferTestTrialData,
+    task=TransferTrialData,
     upstreams=[
         CreateTrialDatasetInventory,
         CreateTrialAsdf,
@@ -61,4 +61,4 @@ trial.add_node(
 )
 
 # goodby
-trial.add_node(task=TrialTeardown, upstreams=TransferTestTrialData)
+trial.add_node(task=TrialTeardown, upstreams=TransferTrialData)

{dkist_processing_test-1.21.3rc1.dist-info → dkist_processing_test-1.23.8rc1.dist-info}/METADATA

@@ -1,8 +1,8 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: dkist-processing-test
-Version: 1.21.3rc1
+Version: 1.23.8rc1
 Summary: Example instrument code used by the DKIST science data processing pipelines to test processing infrastructure
-Author: NSO / AURA
+Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
 Project-URL: Homepage, https://nso.edu/dkist/data-center/
 Project-URL: Repository, https://bitbucket.org/dkistdc/dkist-processing-test/
@@ -12,31 +12,31 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.11
 Description-Content-Type: text/x-rst
-Requires-Dist: dkist-processing-common ==10.2.2rc1
-Requires-Dist: dkist-header-validator ==5.1.1
-Requires-Dist: dkist-service-configuration ==2.2
-Requires-Dist: dkist-fits-specifications ==4.7.0
-Requires-Dist: numba ==0.59.1
-Requires-Dist: astropy ==6.1.0
-Requires-Dist: numpy ==1.26.4
-Requires-Dist: dkist-spectral-lines ==3.0.0
-Provides-Extra: asdf
-Requires-Dist: dkist-processing-common[asdf] ; extra == 'asdf'
-Requires-Dist: dkist-inventory[asdf] ==1.4.0 ; extra == 'asdf'
+Requires-Dist: dkist-processing-common==10.5.6
+Requires-Dist: dkist-header-validator==5.1.1
+Requires-Dist: dkist-service-configuration==2.2
+Requires-Dist: dkist-fits-specifications==4.10.0
+Requires-Dist: numba==0.59.1
+Requires-Dist: astropy==6.1.0
+Requires-Dist: numpy==1.26.4
+Requires-Dist: dkist-spectral-lines==3.0.0
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
+Requires-Dist: dkist-data-simulator>=5.2.0; extra == "test"
+Requires-Dist: dkist-processing-test[inventory]; extra == "test"
+Requires-Dist: dkist-processing-test[asdf]; extra == "test"
+Requires-Dist: dkist-processing-test[quality]; extra == "test"
 Provides-Extra: inventory
-Requires-Dist: dkist-processing-common[inventory] ; extra == 'inventory'
-Requires-Dist: dkist-inventory ==1.4.0 ; extra == 'inventory'
+Requires-Dist: dkist-processing-common[inventory]; extra == "inventory"
+Requires-Dist: dkist-inventory==1.5.0rc1; extra == "inventory"
+Provides-Extra: asdf
+Requires-Dist: dkist-processing-common[asdf]; extra == "asdf"
+Requires-Dist: dkist-inventory[asdf]==1.5.0rc1; extra == "asdf"
 Provides-Extra: quality
-Requires-Dist: dkist-quality ==1.1.1 ; extra == 'quality'
-Provides-Extra: test
-Requires-Dist: pytest ; extra == 'test'
-Requires-Dist: pytest-xdist ; extra == 'test'
-Requires-Dist: pytest-cov ; extra == 'test'
-Requires-Dist: pytest-mock ; extra == 'test'
-Requires-Dist: dkist-data-simulator >=5.2.0 ; extra == 'test'
-Requires-Dist: dkist-processing-test[inventory] ; extra == 'test'
-Requires-Dist: dkist-processing-test[asdf] ; extra == 'test'
-Requires-Dist: dkist-processing-test[quality] ; extra == 'test'
+Requires-Dist: dkist-quality==1.2.0; extra == "quality"
 
 dkist-processing-test
 ---------------------

{dkist_processing_test-1.21.3rc1.dist-info → dkist_processing_test-1.23.8rc1.dist-info}/RECORD

@@ -2,33 +2,33 @@ dkist_processing_test/__init__.py,sha256=t-Daj1wOIg5dgI1gklg6suSnI-xi6DKSb6DxHDW
 dkist_processing_test/config.py,sha256=LF80ReTk0ggJ3eZI4NZRZCgmR6E84adWU5iP6-JN-XY,383
 dkist_processing_test/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dkist_processing_test/models/constants.py,sha256=TxHYZHl3YQXN7-Voj7QgjgAajbxxB_qR8tRI9iTdERI,726
-dkist_processing_test/models/parameters.py,sha256=5lUvGEmPh5JnGeLpJTpzrIao56bpRSyvfFZsRoONwYg,1256
-dkist_processing_test/tasks/__init__.py,sha256=ZWoqlNgFrn2auTMGNX9YWmHW_3JwQzn04XE_FmCAU24,517
+dkist_processing_test/models/parameters.py,sha256=OLrS1O_xZptjdNW882Otx5cHWyd4FH7nzi8fkZnj9GM,1778
+dkist_processing_test/tasks/__init__.py,sha256=mzbjxWY9fdNKUyfXDC_vXuSaB7G0-iwrumCUlq6keoA,457
 dkist_processing_test/tasks/exercise_numba.py,sha256=XfMVffPUgeiPoxSgo39tIjzFBJKKt3fnIYGaDFyBibc,1437
 dkist_processing_test/tasks/fail.py,sha256=jiOiuqoX_JR5hx81REgvcSvb0GBlVKc9MIXKUO0Nhr4,245
-dkist_processing_test/tasks/fake_science.py,sha256=Hon9h5TYc3fuHerm_1uLW2gNr4Y3W0oBN3nmIy4qw-M,4082
+dkist_processing_test/tasks/fake_science.py,sha256=-DMjAtQbtTly0PAv78lIgD8T_HVMo5mmBjyLiYQF0Ko,4555
 dkist_processing_test/tasks/high_memory.py,sha256=J3vBsivFkqs8eZZwBPpEORlHnFX1bGsbO3CjllJMRsc,540
 dkist_processing_test/tasks/manual.py,sha256=gjol_EieMbclv0ZDkVqR2Xd7abutali6KNwoqaCJiAI,583
 dkist_processing_test/tasks/movie.py,sha256=uFPWQlRFP5zzzYGT58IXebbG49su2I4AEpN2bIy7vGc,1831
 dkist_processing_test/tasks/noop.py,sha256=txgpa0DYrjYPKBDfwfjHt7X9Dna4yaJeMCabu3p2Zmo,235
 dkist_processing_test/tasks/parse.py,sha256=4bDKnPaWeZkQbUpYWCWk1O-Fjzu6WWRboFef3NFJDMs,2713
 dkist_processing_test/tasks/quality.py,sha256=9QynI9R5mkYueb46wNBUhK62QSd2n791L0_w6nVxBjc,895
-dkist_processing_test/tasks/trial_output_data.py,sha256=T34kGFxyR0IVaYnJhP3s9kv1_ExSNlnxx0n76cN32NU,1553
 dkist_processing_test/tasks/write_l1.py,sha256=lpNyCI3kH_pnL0Wi1LHhNRjRHNxu6q76xccCNHm7uwA,2074
 dkist_processing_test/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dkist_processing_test/tests/conftest.py,sha256=UhtnrZUgbfX738Ej2MhxZMIjimHnUipfQbGxCLSaH8k,1939
-dkist_processing_test/tests/test_parameters.py,sha256=nh8znKVxFhC46LYNi-DkhV24LFYTfFQI0duV0DwNoHs,3164
-dkist_processing_test/tests/test_tasks.py,sha256=Yvjw8ImQO4QudC5F969LQZMoFT_959CYsldFnZh5r10,25388
+dkist_processing_test/tests/conftest.py,sha256=9H8_tqNKofHnl1IRqYM2-M2_K_TJ3b6oaIfb1dVkbsg,2728
+dkist_processing_test/tests/parameter_models.py,sha256=c-SqCBqHP4KkFs0_Y7HLED4mH4oMCpAYu-rYVlTHy1M,4635
+dkist_processing_test/tests/test_parameters.py,sha256=KGeEUOhtC_l-LN6hjB2jgwUZIWAMBx8wlmEeIja-eKo,4572
+dkist_processing_test/tests/test_tasks.py,sha256=woNEolAfahLowR7T9nTaZgNJidkGmGpfU6Xyc4LXbqQ,26857
 dkist_processing_test/tests/test_workflows.py,sha256=NqRkstqcqwH80cKJ1uDw16G4MsDOuOiZRWdfH0STarQ,286
 dkist_processing_test/workflows/__init__.py,sha256=Ryn_HXsVkZM_lLKyVteXC909Td2geylmz_1Z-8SE8c8,78
-dkist_processing_test/workflows/common_tasks.py,sha256=4H1TaUNa5-HyE1NJQiAc3NAN62ZOTYb7MbR1yjKMg-k,6843
+dkist_processing_test/workflows/common_tasks.py,sha256=UpUTt-QhD-jkzBB3p1ECRbtqY3wjP6Lfsy4TdvxZKQ8,6837
 dkist_processing_test/workflows/end_to_end.py,sha256=XzSe7XtPF0e3fLXWL90LOZ0WKCoCKA4SL43N0CzgpCQ,2328
 dkist_processing_test/workflows/exercise_numba.py,sha256=phpPYcIXEsmc0X1yls_YbXoKS6VXxhHe_8vNd_GyajU,347
 dkist_processing_test/workflows/fail.py,sha256=KGhyAF7gKYYutP4aGa-1tDqPsC7Nr4xKYY0z-HByWi0,326
 dkist_processing_test/workflows/noop.py,sha256=k2-6BpRkl1JDGaHPavxDzIlVx11KQtxKmsHmHTNQ9o0,666
 dkist_processing_test/workflows/resource_queue.py,sha256=_jOT99O0MFN_ACne6i8T5ZIwe_lBQqGz2Ccb8JlqQMI,500
-dkist_processing_test/workflows/trial_end_to_end.py,sha256=wkl_gk0zQ5TJ41CY0gEREPSIldZdTtLGwPDL_iwObe4,2425
-dkist_processing_test-1.21.3rc1.dist-info/METADATA,sha256=VPKAoUacF_IyHXh0vT4vbORjlu0zWX_ErDBmX2ddhbQ,3274
-dkist_processing_test-1.21.3rc1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-dkist_processing_test-1.21.3rc1.dist-info/top_level.txt,sha256=Hs4oTIrG_r-svhk_RGFTEO4e3vqQoYlBzdv5mvJVF24,22
-dkist_processing_test-1.21.3rc1.dist-info/RECORD,,
+dkist_processing_test/workflows/trial_end_to_end.py,sha256=4QSphBegbetDben5PdA5_J64hWJVV8ciVdF90qkyZCo,2415
+dkist_processing_test-1.23.8rc1.dist-info/METADATA,sha256=qknfhMMDgkZuP1az_oncgF5cizxLlmIw_b7NX96Ys8Y,3277
+dkist_processing_test-1.23.8rc1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+dkist_processing_test-1.23.8rc1.dist-info/top_level.txt,sha256=Hs4oTIrG_r-svhk_RGFTEO4e3vqQoYlBzdv5mvJVF24,22
+dkist_processing_test-1.23.8rc1.dist-info/RECORD,,

{dkist_processing_test-1.21.3rc1.dist-info → dkist_processing_test-1.23.8rc1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.1.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

dkist_processing_test/tasks/trial_output_data.py

@@ -1,46 +0,0 @@
-"""Trial tasks"""
-from dkist_processing_common.tasks.mixin.globus import GlobusTransferItem
-from dkist_processing_common.tasks.trial_output_data import TransferTrialDataBase
-from dkist_service_configuration.logging import logger
-
-__all__ = ["TransferTestTrialData"]
-
-
-class TransferTestTrialData(TransferTrialDataBase):
-    @property
-    def intermediate_task_names(self) -> list[str]:
-        """Return a list of intermediate tasks we want to transfer.
-
-        Just a dummy task for testing.
-        """
-        return ["DUMMY"]
-
-    def build_transfer_list(self) -> list[GlobusTransferItem]:
-        """
-        Build a list containing all files we want to transfer to the trial environment.
-
-        For the purposes of testing we try to exercise all of the provided helper methods.
-        """
-        transfer_list = []
-
-        transfer_list += self.build_debug_frame_transfer_list()
-
-        transfer_list += self.build_intermediate_frame_transfer_list()
-
-        transfer_list += self.build_output_frame_transfer_list()
-
-        transfer_list += self.build_output_dataset_inventory_transfer_list()
-
-        transfer_list += self.build_output_asdf_transfer_list()
-
-        transfer_list += self.build_output_quality_data_transfer_list()
-
-        transfer_list += self.build_output_quality_report_transfer_list()
-
-        transfer_list += self.build_output_movie_transfer_list()
-
-        transfer_list += self.build_transfer_list_from_tag_lists([["FOO", "BAR"], ["BAZ"]])
-
-        logger.info(f"{transfer_list = }")
-
-        return transfer_list