dkist-processing-common 10.5.4__py3-none-any.whl → 12.1.0rc1__py3-none-any.whl

dkist_processing_common/tests/test_submit_dataset_metadata.py

@@ -7,7 +7,6 @@ from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks import SubmitDatasetMetadata
 from dkist_processing_common.tasks.mixin import metadata_store
-from dkist_processing_common.tests.conftest import FakeGQLClient
 
 
 @pytest.fixture()
@@ -86,6 +85,7 @@ def submit_dataset_metadata_task(
 def test_submit_dataset_metadata(
     submit_dataset_metadata_task,
     mocker,
+    fake_gql_client,
 ):
     """
     :Given: An instance of SubmitDatasetMetadata with tagged processed data
@@ -93,15 +93,12 @@ def test_submit_dataset_metadata(
     :Then: Metadata files for the dataset are saved to the remote database
     """
     mocker.patch(
-        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
     )
-    # intercept these two GraphQLClient calls so they can be confirmed
+    # intercept this GraphQLClient call so it can be confirmed
     mocked_metadata_store_add_dataset_receipt_account = mocker.patch.object(
         metadata_store.MetadataStoreMixin, "metadata_store_add_dataset_receipt_account"
     )
-    mocked_metadata_store_add_quality_data = mocker.patch.object(
-        metadata_store.MetadataStoreMixin, "metadata_store_add_quality_data"
-    )
     task = submit_dataset_metadata_task
 
     # When
@@ -109,4 +106,3 @@ def test_submit_dataset_metadata(
 
     # Then
     mocked_metadata_store_add_dataset_receipt_account.assert_called_once()
-    mocked_metadata_store_add_quality_data.assert_called_once()

dkist_processing_common/tests/test_tags.py

@@ -1,6 +1,7 @@
 """
 Tests for the tag cloud manager
 """
+
 import random
 from typing import Callable
 from uuid import uuid4

dkist_processing_common/tests/test_task_name.py

@@ -19,7 +19,7 @@ def test_task_name_tags():
     assert Tag.task_geometric() == f"TASK_{TaskName.geometric.value}"
     assert Tag.task_geometric_angle() == f"TASK_{TaskName.geometric_angle.value}"
     assert (
-        Tag.task_geometric_sepectral_shifts() == f"TASK_{TaskName.geometric_spectral_shifts.value}"
+        Tag.task_geometric_spectral_shifts() == f"TASK_{TaskName.geometric_spectral_shifts.value}"
     )
     assert Tag.task_geometric_offset() == f"TASK_{TaskName.geometric_offsets.value}"
     assert Tag.task_polcal() == f"TASK_{TaskName.polcal.value}"

dkist_processing_common/tests/test_task_parsing.py

@@ -1,3 +1,5 @@
+from enum import StrEnum
+
 import pytest
 from astropy.io import fits
 
@@ -9,6 +11,15 @@ from dkist_processing_common.parsers.task import parse_polcal_task_type
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 
 
+class DummyMetadataKey(StrEnum):
+    ip_task_type = "IPTASK"
+    gos_level3_status = "GOSLVL3"
+    gos_level3_lamp_status = "GOSLAMP"
+    gos_level0_status = "GOSLVL0"
+    gos_retarder_status = "GOSRET"
+    gos_polarizer_status = "GOSPOL"
+
+
 class DummyFitsAccess(FitsAccessBase):
     def __init__(
         self,
@@ -17,13 +28,12 @@ class DummyFitsAccess(FitsAccessBase):
         auto_squeeze: bool = False,  # Because L1 data should always have the right form, right?
     ):
         super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
-
-        self.ip_task_type: str = self.header["IPTASK"]
-        self.gos_level3_status: str = self.header["GOSLVL3"]
-        self.gos_level3_lamp_status: str = self.header["GOSLAMP"]
-        self.gos_level0_status: str = self.header["GOSLVL0"]
-        self.gos_retarder_status: str = self.header["GOSRET"]
-        self.gos_polarizer_status: str = self.header["GOSPOL"]
+        self.ip_task_type = self.header[DummyMetadataKey.ip_task_type]
+        self.gos_level3_status = self.header[DummyMetadataKey.gos_level3_status]
+        self.gos_level3_lamp_status = self.header[DummyMetadataKey.gos_level3_lamp_status]
+        self.gos_level0_status = self.header[DummyMetadataKey.gos_level0_status]
+        self.gos_retarder_status = self.header[DummyMetadataKey.gos_retarder_status]
+        self.gos_polarizer_status = self.header[DummyMetadataKey.gos_polarizer_status]
 
 
 @pytest.fixture

dkist_processing_common/tests/test_teardown.py

@@ -5,10 +5,11 @@ import pytest
 
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.str import str_encoder
-from dkist_processing_common.models.graphql import RecipeRunResponse
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.teardown import Teardown
-from dkist_processing_common.tests.conftest import FakeGQLClient
+from dkist_processing_common.tests.mock_metadata_store import RecipeRunResponseMapping
+from dkist_processing_common.tests.mock_metadata_store import fake_gql_client_factory
+from dkist_processing_common.tests.mock_metadata_store import make_default_recipe_run_response
 
 
 class TeardownTest(Teardown):
@@ -18,33 +19,36 @@ class TeardownTest(Teardown):
 
 @pytest.fixture()
 def make_mock_GQL_with_configuration():
-    def class_generator(configuration: dict):
-        class TeardownFakeGQLClient(FakeGQLClient):
-            def execute_gql_query(self, **kwargs):
-                response = super().execute_gql_query(**kwargs)
-                if isinstance(response, list):
-                    if isinstance(response[0], RecipeRunResponse):
-                        response[0].configuration = json.dumps(configuration)
-                return response
-
+    def class_generator(teardown_option: bool | None):
+        recipe_run_response = make_default_recipe_run_response()
+        config = recipe_run_response.configuration
+        if isinstance(teardown_option, bool):
+            config.teardown_enabled = teardown_option
+        else:
+            config_dict = config.model_dump(exclude="teardown_enabled")
+            config = json.dumps(config_dict)
+        response_mapping_override = RecipeRunResponseMapping(response=recipe_run_response)
+        TeardownFakeGQLClient = fake_gql_client_factory(
+            response_mapping_override=response_mapping_override
+        )
         return TeardownFakeGQLClient
 
     return class_generator
 
 
 @pytest.fixture(scope="session")
-def config_with_teardown_enabled() -> dict:
-    return {"teardown_enabled": True}
+def teardown_enabled() -> bool:
+    return True
 
 
 @pytest.fixture(scope="session")
-def config_with_teardown_disabled() -> dict:
-    return {"teardown_enabled": False}
+def teardown_disabled() -> bool:
+    return False
 
 
 @pytest.fixture(scope="session")
-def config_with_no_teardown() -> dict:
-    return dict()
+def teardown_default() -> None:
+    return None
 
 
 @pytest.fixture(scope="function")
@@ -75,14 +79,14 @@ def teardown_task_factory(tmp_path, recipe_run_id):
 
 
 def test_purge_data(
-    teardown_task_factory, make_mock_GQL_with_configuration, config_with_teardown_enabled, mocker
+    teardown_task_factory, make_mock_GQL_with_configuration, teardown_enabled, mocker
 ):
     """
     :Given: A Teardown task with files and tags linked to it and teardown enabled
     :When: Running the task
     :Then: All the files are deleted and the tags are removed
     """
-    FakeGQLClass = make_mock_GQL_with_configuration(config_with_teardown_enabled)
+    FakeGQLClass = make_mock_GQL_with_configuration(teardown_enabled)
     mocker.patch(
         "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClass
     )
@@ -102,14 +106,14 @@ def test_purge_data(
 
 
 def test_purge_data_disabled(
-    teardown_task_factory, make_mock_GQL_with_configuration, config_with_teardown_disabled, mocker
+    teardown_task_factory, make_mock_GQL_with_configuration, teardown_disabled, mocker
 ):
     """
     :Given: A Teardown task with files and tags linked to it and teardown disabled
     :When: Running the task
     :Then: All the files are not deleted and the tags remain
     """
-    FakeGQLClass = make_mock_GQL_with_configuration(config_with_teardown_disabled)
+    FakeGQLClass = make_mock_GQL_with_configuration(teardown_disabled)
     mocker.patch(
         "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClass
     )
@@ -129,14 +133,14 @@ def test_purge_data_disabled(
 
 
 def test_purge_data_no_config(
-    teardown_task_factory, make_mock_GQL_with_configuration, config_with_no_teardown, mocker
+    teardown_task_factory, make_mock_GQL_with_configuration, teardown_default, mocker
 ):
     """
-    :Given: A Teardown task with files and tags linked and teardown not specified in the configuration
+    :Given: A Teardown task with files and tags linked and default teardown configuration
     :When: Running the task
     :Then: All the files are deleted and the tags are removed
     """
-    FakeGQLClass = make_mock_GQL_with_configuration(config_with_no_teardown)
+    FakeGQLClass = make_mock_GQL_with_configuration(teardown_default)
     mocker.patch(
         "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClass
     )

dkist_processing_common/tests/test_transfer_input_data.py

@@ -1,167 +1,312 @@
 import json
 import os
-from itertools import chain
 from pathlib import Path
-from uuid import uuid4
 
 import pytest
 
 from dkist_processing_common._util.scratch import WorkflowFileSystem
+from dkist_processing_common.codecs.basemodel import basemodel_decoder
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.tasks import WorkflowTaskBase
 from dkist_processing_common.tasks.transfer_input_data import TransferL0Data
-from dkist_processing_common.tests.conftest import create_parameter_files
+from dkist_processing_common.tests.mock_metadata_store import InputDatasetRecipeRunResponseMapping
+from dkist_processing_common.tests.mock_metadata_store import default_calibration_frames_doc
+from dkist_processing_common.tests.mock_metadata_store import default_observe_frames_doc
+from dkist_processing_common.tests.mock_metadata_store import default_parameters_doc
+from dkist_processing_common.tests.mock_metadata_store import fake_gql_client_factory
+from dkist_processing_common.tests.mock_metadata_store import (
+    make_default_input_dataset_recipe_run_response,
+)
+
+
+def create_parameter_files(
+    task: WorkflowTaskBase, parameters_doc: list[dict] = default_parameters_doc
+):
+    """
+    Create the parameter files specified in the parameters document returned by the metadata store.
+
+    This fixture assumes that the JSON parameters document has already been loaded into a python
+    structure (a list of dicts), but the parameter values themselves are still JSON.
+    """
+    for parameter in parameters_doc:
+        for value in parameter["parameterValues"]:
+            if "__file__" not in value["parameterValue"]:
+                continue
+            parameter_value = json.loads(value["parameterValue"])
+            param_path = parameter_value["__file__"]["objectKey"]
+            file_path = task.scratch.workflow_base_path / Path(param_path)
+            if not file_path.parent.exists():
+                file_path.parent.mkdir(parents=True, exist_ok=True)
+            file_path.write_text(data="")
+            task.tag(path=file_path, tags=Tag.parameter(param_path))
+
+
+def create_input_frames(
+    task: WorkflowTaskBase,
+    input_frame_docs: list[dict] = default_observe_frames_doc + default_calibration_frames_doc,
+):
+    """
+    Create the observe and calibration frame files specified in the input dataset documents
+    returned by the metadata store.
+    """
+    for frame in input_frame_docs:
+        for object_key in frame["object_keys"]:
+            file_path = task.scratch.workflow_base_path / Path(object_key)
+            if not file_path.parent.exists():
+                file_path.parent.mkdir(parents=True, exist_ok=True)
+            file_path.write_text(data="")
+            task.tag(path=file_path, tags=[Tag.frame(), Tag.input()])
+
+
+class TransferL0DataTask(TransferL0Data):
+    def run(self) -> None: ...
 
 
 @pytest.fixture
-def transfer_l0_data(recipe_run_id, tmp_path) -> dict:
-    task = TransferL0Data(
+def fake_gql_client_class_missing_calibration_part():
+    input_dataset_recipe_run_response = make_default_input_dataset_recipe_run_response()
+    dataset_parts = (
+        input_dataset_recipe_run_response.recipeInstance.inputDataset.inputDatasetInputDatasetParts
+    )
+    for index, part in enumerate(dataset_parts):
+        if (
+            part.inputDatasetPart.inputDatasetPartType.inputDatasetPartTypeName
+            == "calibration_frames"
+        ):
+            del dataset_parts[index]
+    new_response_mapping = InputDatasetRecipeRunResponseMapping(
+        response=input_dataset_recipe_run_response
+    )
+    FakeGQLClientMissingInputDatasetCalibrationPart = fake_gql_client_factory(
+        response_mapping_override=new_response_mapping
+    )
+
+    return FakeGQLClientMissingInputDatasetCalibrationPart
+
+
+def _transfer_l0_data_task_with_client(recipe_run_id, tmp_path, mocker, client_cls):
+    mocker.patch(
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
+        new=client_cls,
+    )
+    with TransferL0DataTask(
         recipe_run_id=recipe_run_id,
         workflow_name="workflow_name",
         workflow_version="workflow_version",
+    ) as task:
+        task.scratch = WorkflowFileSystem(
+            recipe_run_id=recipe_run_id,
+            scratch_base_path=tmp_path,
+        )
+        yield task
+        task._purge()
+
+
+@pytest.fixture
+def transfer_l0_data_task(recipe_run_id, tmp_path, mocker, fake_gql_client):
+    yield from _transfer_l0_data_task_with_client(recipe_run_id, tmp_path, mocker, fake_gql_client)
+
+
+@pytest.fixture
+def transfer_l0_data_task_missing_calibration_part(
+    recipe_run_id, tmp_path, mocker, fake_gql_client_class_missing_calibration_part
+):
+    yield from _transfer_l0_data_task_with_client(
+        recipe_run_id, tmp_path, mocker, fake_gql_client_class_missing_calibration_part
     )
-    task.scratch = WorkflowFileSystem(
-        recipe_run_id=recipe_run_id,
-        scratch_base_path=tmp_path,
-    )
-    task.scratch.scratch_base_path = tmp_path
-    input_dataset_parameters_part = [
-        {
-            "parameterName": "param_name_1",
-            "parameterValues": [
-                {
-                    "parameterValueId": 1,
-                    "parameterValue": json.dumps([[1, 2, 3], [4, 5, 6], [7, 8, 9]]),
-                    "parameterValueStartDate": "2000-01-01",
-                }
-            ],
-        },
-        {
-            "parameterName": "param_name_2",
-            "parameterValues": [
-                {
-                    "parameterValueId": 2,
-                    "parameterValue": json.dumps(
-                        {
-                            "__file__": {
-                                "bucket": "data",
-                                "objectKey": f"parameters/param_name/{uuid4().hex}.dat",
-                            }
-                        }
-                    ),
-                    "parameterValueStartDate": "2000-01-01",
-                },
-                {
-                    "parameterValueId": 3,
-                    "parameterValue": json.dumps(
-                        {
-                            "__file__": {
-                                "bucket": "data",
-                                "objectKey": f"parameters/param_name/{uuid4().hex}.dat",
-                            }
-                        }
-                    ),
-                    "parameterValueStartDate": "2000-01-02",
-                },
-            ],
-        },
-        {
-            "parameterName": "param_name_4",
-            "parameterValues": [
-                {
-                    "parameterValueId": 4,
-                    "parameterValue": json.dumps(
-                        {"a": 1, "b": 3.14159, "c": "foo", "d": [1, 2, 3]}
-                    ),
-                    "parameterValueStartDate": "2000-01-01",
-                }
-            ],
-        },
-    ]
-    input_dataset_observe_frames_part = [
-        {
-            "bucket": uuid4().hex[:6],
-            "object_keys": [Path(uuid4().hex[:6]).as_posix() for _ in range(3)],
-        }
-    ]
-    input_dataset_calibration_frames_part = [
-        {
-            "bucket": uuid4().hex[:6],
-            "object_keys": [Path(uuid4().hex[:6]).as_posix() for _ in range(3)],
-        },
-        {
-            "bucket": uuid4().hex[:6],
-            "object_keys": [Path(uuid4().hex[:6]).as_posix() for _ in range(3)],
-        },
-    ]
-    # load parameters file
-    file_path = task.scratch.workflow_base_path / Path(f"{uuid4().hex[:6]}.ext")
-    file_path.write_text(data=json.dumps(input_dataset_parameters_part))
-    task.tag(path=file_path, tags=Tag.input_dataset_parameters())
-    # create parameter files
-    expected_parameters = dict()
-    for item in input_dataset_parameters_part:
-        expected_parameters[item["parameterName"]] = item["parameterValues"]
-    create_parameter_files(task, expected_parameters)
-    # load observe frames file
-    file_path = task.scratch.workflow_base_path / Path(f"{uuid4().hex[:6]}.ext")
-    file_path.write_text(data=json.dumps(input_dataset_observe_frames_part))
-    task.tag(path=file_path, tags=Tag.input_dataset_observe_frames())
-    # load calibration frames file
-    file_path = task.scratch.workflow_base_path / Path(f"{uuid4().hex[:6]}.ext")
-    file_path.write_text(data=json.dumps(input_dataset_calibration_frames_part))
-    task.tag(path=file_path, tags=Tag.input_dataset_calibration_frames())
-
-    yield {
-        "task": task,
-        "parameters": input_dataset_parameters_part,
-        "observe": input_dataset_observe_frames_part,
-        "calibration": input_dataset_calibration_frames_part,
-    }
-    task._purge()
-
-
-def test_format_frame_transfer_items(transfer_l0_data):
+
+
+@pytest.mark.parametrize(
+    "expected_doc, tag",
+    [
+        pytest.param(
+            default_observe_frames_doc,
+            Tag.input_dataset_observe_frames(),
+            id="observe_frames",
+        ),
+        pytest.param(
+            default_calibration_frames_doc,
+            Tag.input_dataset_calibration_frames(),
+            id="calibration_frames",
+        ),
+        pytest.param(
+            default_parameters_doc,
+            Tag.input_dataset_parameters(),
+            id="parameters",
+        ),
+    ],
+)
+def test_download_dataset(transfer_l0_data_task, expected_doc, tag):
     """
     :Given: a TransferL0Data task with a valid input dataset
-    :When: formatting items in the input dataset for transfer
-    :Then: the items are correctly loaded into GlobusTransferItem objects
+    :When: downloading the dataset documents from the metadata store
+    :Then: the correct documents are written to disk, along with tags for file parameters
+    """
+    # Given
+    task = transfer_l0_data_task
+    # When
+    task.download_input_dataset()
+    # Then
+    doc_from_file = next(
+        task.read(tags=tag, decoder=basemodel_decoder, model=InputDatasetPartDocumentList)
+    )
+    doc_list_from_file = doc_from_file.model_dump()["doc_list"]
+    if (
+        tag == Tag.input_dataset_parameters()
+    ):  # parameter doc gets written with tags for file objects
+        for item in expected_doc:
+            for val in item["parameterValues"]:
+                if "__file__" in val["parameterValue"]:
+                    file_dict = json.loads(val["parameterValue"])["__file__"]
+                    file_dict["tag"] = Tag.parameter(Path(file_dict["objectKey"]).name)
+                    val["parameterValue"] = json.dumps({"__file__": file_dict})
+    assert doc_list_from_file == expected_doc
+
+
+def test_download_dataset_missing_part(transfer_l0_data_task_missing_calibration_part):
     """
-    task = transfer_l0_data["task"]
+    :Given: a TransferL0Data task with a valid input dataset without calibration frames
+    :When: downloading the dataset documents from the metadata store
+    :Then: the correct number of documents are written to disk
+    """
+    # Given
+    task = transfer_l0_data_task_missing_calibration_part
+    # When
+    task.download_input_dataset()
+    # Then
+    observe_doc_from_file = next(
+        task.read(
+            tags=Tag.input_dataset_observe_frames(),
+            decoder=basemodel_decoder,
+            model=InputDatasetPartDocumentList,
+        )
+    )
+    parameters_doc_from_file = next(
+        task.read(
+            tags=Tag.input_dataset_parameters(),
+            decoder=basemodel_decoder,
+            model=InputDatasetPartDocumentList,
+        )
+    )
+    with pytest.raises(StopIteration):
+        calibration_doc_from_file = next(
+            task.read(
+                tags=Tag.input_dataset_calibration_frames(),
+                decoder=basemodel_decoder,
+                model=InputDatasetPartDocumentList,
+            )
+        )
+
+
+@pytest.mark.parametrize(
+    "task_name",
+    [
+        pytest.param(
+            "transfer_l0_data_task",
+            id="observe_and_calibration_frames",
+        ),
+        pytest.param(
+            "transfer_l0_data_task_missing_calibration_part",
+            id="calibration_frames_missing",
+        ),
+    ],
+)
+def test_build_frame_transfer_list_formatted(request, task_name):
+    """
+    :Given: a TransferL0Data task with downloaded input dataset docs
+    :When: building a list of frames in the input dataset formatted for transfer
+    :Then: the correct items are correctly loaded into GlobusTransferItem objects
+    """
+    # Given
+    task = request.getfixturevalue(task_name)
+    task.download_input_dataset()
+    # When
+    observe_transfer_objects = task.build_transfer_list(doc_tag=Tag.input_dataset_observe_frames())
+    calibration_transfer_objects = task.build_transfer_list(
+        doc_tag=Tag.input_dataset_calibration_frames()
+    )
+    transfer_objects = observe_transfer_objects + calibration_transfer_objects
+    formatted_transfer_items = task.format_transfer_items(input_dataset_objects=transfer_objects)
+    # Then
     source_filenames = []
     destination_filenames = []
-    for frame_set in chain(transfer_l0_data["observe"], transfer_l0_data["calibration"]):
+    expected_frames = list(default_observe_frames_doc)
+    if "missing_calibration_part" not in task_name:
+        expected_frames += default_calibration_frames_doc
+    for frame_set in expected_frames:
         for key in frame_set["object_keys"]:
             source_filenames.append(os.path.join("/", frame_set["bucket"], key))
             destination_filenames.append(Path(key).name)
-    assert len(task.format_frame_transfer_items()) == len(source_filenames)
-    for item in task.format_frame_transfer_items():
+    assert len(formatted_transfer_items) == len(source_filenames)
+    for item in formatted_transfer_items:
         assert item.source_path.as_posix() in source_filenames
         assert item.destination_path.name in destination_filenames
         assert not item.recursive
 
 
-def test_format_parameter_file_transfer_items(transfer_l0_data):
+def test_build_parameter_file_transfer_items(transfer_l0_data_task):
     """
-    :Given: a TransferL0Data task with a valid input dataset
-    :When: formatting items in the input dataset for transfer
-    :Then: the items are correctly loaded into GlobusTransferItem objects
+    :Given: a TransferL0Data task with downloaded input dataset docs
+    :When: building a list of parameter files formatted for transfer
+    :Then: the correct items are correctly loaded into GlobusTransferItem objects
     """
-    task = transfer_l0_data["task"]
+    # Given
+    task = transfer_l0_data_task
+    task.download_input_dataset()
+    # When
+    transfer_objects = task.build_transfer_list(doc_tag=Tag.input_dataset_parameters())
+    formatted_transfer_items = task.format_transfer_items(input_dataset_objects=transfer_objects)
+    # Then
     source_filenames = []
     destination_filenames = []
-    for param in transfer_l0_data["parameters"]:
+    parameters = default_parameters_doc
+    for param in parameters:
         for value in param["parameterValues"]:
-            param_value = value["parameterValue"]
-            if "__file__" in param_value:
-                value_dict = json.loads(param_value)
+            if "__file__" in value["parameterValue"]:
+                value_dict = json.loads(value["parameterValue"])
                 bucket = value_dict["__file__"]["bucket"]
                 object_key = value_dict["__file__"]["objectKey"]
                 source_filenames.append(os.path.join("/", bucket, object_key))
                 destination_filenames.append(Path(object_key).name)
-    transfer_items = task.format_parameter_transfer_items()
-    assert len(transfer_items) == len(source_filenames)
-    for transfer_item in transfer_items:
+    assert len(formatted_transfer_items) == len(source_filenames)
+    for transfer_item in formatted_transfer_items:
         assert transfer_item.source_path.as_posix() in source_filenames
         assert transfer_item.destination_path.name in destination_filenames
         assert str(transfer_item.destination_path).startswith(str(task.scratch.workflow_base_path))
         assert not transfer_item.recursive
+
+
+def test_tag_transfer_items(transfer_l0_data_task):
+    """
+    :Given: a TransferL0Data task with downloaded input dataset frames and parameter files
+    :When: tagging the downloaded files
+    :Then: the downloaded items are correctly tagged
+    """
+    # Given
+    task = transfer_l0_data_task
+    task.download_input_dataset()
+    observe_transfer_objects = task.build_transfer_list(doc_tag=Tag.input_dataset_observe_frames())
+    calibration_transfer_objects = task.build_transfer_list(
+        doc_tag=Tag.input_dataset_calibration_frames()
+    )
+    frame_transfer_objects = observe_transfer_objects + calibration_transfer_objects
+    create_input_frames(task)
+    parameter_transfer_objects = task.build_transfer_list(doc_tag=Tag.input_dataset_parameters())
+    create_parameter_files(task)
+    # When
+    transfer_objects = frame_transfer_objects + parameter_transfer_objects
+    task.tag_transfer_objects(input_dataset_objects=transfer_objects)
+    # Then
+    input_tags = [Tag.input(), Tag.frame()]
+    input_frames_on_disk = list(task.scratch.find_all(tags=input_tags))
+    for obj in frame_transfer_objects:
+        destination_path = task.scratch.absolute_path(obj.object_key)
+        assert destination_path in input_frames_on_disk
+    assert len(input_frames_on_disk) == len(frame_transfer_objects)
+    for obj in parameter_transfer_objects:
+        destination_path = task.scratch.absolute_path(obj.object_key)
+        param_tag = Tag.parameter(Path(obj.object_key))
+        param_file_on_disk = list(task.scratch.find_all(tags=param_tag))
+        assert destination_path in param_file_on_disk
+        assert len(param_file_on_disk) == 1