dkist-processing-common 10.5.4__py3-none-any.whl → 12.1.0rc1__py3-none-any.whl

dkist_processing_common/tests/test_parameters.py

@@ -7,6 +7,7 @@ HOW TO WRITE TESTS FOR NEW PARAMETER SUBCLASSES :
 4. Add a `pytest.param` with this helper function to `test_parameters` to make sure none of the default stuff broke
 5. Write a new test that only uses the helper function to test the new functionality.
 """
+
 import json
 from datetime import datetime
 from datetime import timedelta
@@ -18,22 +19,21 @@ import numpy as np
 import pytest
 from astropy.io import fits
 
+from dkist_processing_common.codecs.array import array_encoder
+from dkist_processing_common.codecs.fits import fits_hdulist_encoder
 from dkist_processing_common.models.parameters import ParameterArmIdMixin
 from dkist_processing_common.models.parameters import ParameterBase
 from dkist_processing_common.models.parameters import ParameterWavelengthMixin
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks import WorkflowTaskBase
-from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetMixin
+from dkist_processing_common.tests.test_input_dataset import input_dataset_frames_part_factory
+
+FITS_FILE = "fits.dat"
+NP_FILE = "np.npy"
 
 
 @pytest.fixture
-def input_dataset_parameters(tmp_path):
-    fits_file_path = tmp_path / "fits.dat"
-    phdu = fits.PrimaryHDU(np.ones((3, 3)) * 3)
-    ihdu = fits.ImageHDU(np.ones((4, 4)) * 4)
-    fits.HDUList([phdu, ihdu]).writeto(fits_file_path)
-    np_file_path = tmp_path / "np.npy"
-    np.save(np_file_path, np.ones((3, 3)) * 4)
+def input_dataset_parameters():
     return [
         {
             "parameterName": "basic_param",
@@ -106,7 +106,15 @@ def input_dataset_parameters(tmp_path):
             "parameterValues": [
                 {
                     "parameterValueId": 1,
-                    "parameterValue": json.dumps({"param_path": str(fits_file_path)}),
+                    "parameterValue": json.dumps(
+                        {
+                            "__file__": {
+                                "bucket": "not_used",
+                                "objectKey": "not_used",
+                                "tag": Tag.parameter(FITS_FILE),
+                            }
+                        }
+                    ),
                 }
             ],
         },
@@ -115,7 +123,15 @@ def input_dataset_parameters(tmp_path):
             "parameterValues": [
                 {
                     "parameterValueId": 1,
-                    "parameterValue": json.dumps({"param_path": str(np_file_path)}),
+                    "parameterValue": json.dumps(
+                        {
+                            "__file__": {
+                                "bucket": "not_used",
+                                "objectKey": "not_used",
+                                "tag": Tag.parameter(NP_FILE),
+                            }
+                        }
+                    ),
                 }
             ],
         },
@@ -129,16 +145,14 @@ def input_dataset_parts(input_dataset_parameters) -> tuple[Any, str]:
 
 @pytest.fixture()
 def task_class_with_parameters(parameter_class) -> Type[WorkflowTaskBase]:
-    class TaskWithParameters(WorkflowTaskBase, InputDatasetMixin):
+    class TaskWithParameters(WorkflowTaskBase):
         def __init__(self, recipe_run_id: int, workflow_name: str, workflow_version: str):
             super().__init__(
                 recipe_run_id=recipe_run_id,
                 workflow_name=workflow_name,
                 workflow_version=workflow_version,
             )
-            self.parameters = parameter_class(
-                input_dataset_parameters=self.input_dataset_parameters
-            )
+            self.parameters = parameter_class(scratch=self.scratch)
 
         def run(self) -> None:
             pass
@@ -149,11 +163,21 @@ def task_class_with_parameters(parameter_class) -> Type[WorkflowTaskBase]:
 @pytest.fixture()
 def task_with_parameters(task_with_input_dataset, task_class_with_parameters):
     task_class = task_class_with_parameters
-    return task_class(
+    with task_class(
         recipe_run_id=task_with_input_dataset.recipe_run_id,
         workflow_name=task_with_input_dataset.workflow_name,
         workflow_version=task_with_input_dataset.workflow_version,
-    )
+    ) as task:
+        phdu = fits.PrimaryHDU(np.ones((3, 3)) * 3)
+        ihdu = fits.ImageHDU(np.ones((4, 4)) * 4)
+        task.write(
+            data=fits.HDUList([phdu, ihdu]),
+            tags=Tag.parameter(FITS_FILE),
+            encoder=fits_hdulist_encoder,
+        )
+        task.write(data=np.ones((3, 3)) * 4, tags=Tag.parameter(NP_FILE), encoder=array_encoder)
+        yield task
+        task._purge()
 
 
 class FilledParametersNoObsTime(ParameterBase):
@@ -182,18 +206,18 @@ class FilledParametersWithObsTime(ParameterBase):
 
     @property
     def fits_file_parameter(self):
-        param_dict = self._find_most_recent_past_value("fits_file_parameter")
-        return self._load_param_value_from_fits(param_dict)
+        param_obj = self._find_most_recent_past_value("fits_file_parameter")
+        return self._load_param_value_from_fits(param_obj=param_obj)
 
     @property
     def non_primary_fits_file_parameter(self):
-        param_dict = self._find_most_recent_past_value("fits_file_parameter")
-        return self._load_param_value_from_fits(param_dict, hdu=1)
+        param_obj = self._find_most_recent_past_value("fits_file_parameter")
+        return self._load_param_value_from_fits(param_obj=param_obj, hdu=1)
 
     @property
     def numpy_file_parameter(self):
-        param_dict = self._find_most_recent_past_value("numpy_file_parameter")
-        return self._load_param_value_from_numpy_save(param_dict)
+        param_obj = self._find_most_recent_past_value("numpy_file_parameter")
+        return self._load_param_value_from_numpy_save(param_obj=param_obj)
 
 
 def parameter_class_with_obs_ip_start_time():
@@ -343,3 +367,28 @@ def test_mixins_error_with_no_arg():
     """
     with pytest.raises(TypeError):
         parameters = FilledWavelengthParameters(input_dataset_parameters={"foo": []})
+
+
+@pytest.mark.parametrize(
+    ("input_dataset_parts", "parameter_class"),
+    [
+        pytest.param(
+            [
+                (input_dataset_frames_part_factory(), Tag.input_dataset_parameters()),
+                (input_dataset_frames_part_factory(), Tag.input_dataset_parameters()),
+            ],
+            parameter_class_with_obs_ip_start_time(),
+            id="two_param_docs",
+        ),
+    ],
+)
+def test_multiple_input_dataset_parameter_parts(
+    request, input_dataset_parts: list[tuple[Any, str]], parameter_class
+):
+    """
+    Given: a task with multiple tagged input dataset parameter docs
+    When: initializing the parameter base
+    Then: an error is raised
+    """
+    with pytest.raises(ValueError, match="more than one parameter file"):
+        request.getfixturevalue("task_with_parameters")

dkist_processing_common/tests/test_parse_l0_input_data.py

@@ -1,4 +1,7 @@
 """Tests for the parse L0 input data task"""
+
+from enum import StrEnum
+
 import numpy as np
 import pytest
 from astropy.io import fits
@@ -9,16 +12,20 @@ from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
 from dkist_processing_common.models.constants import BudName
 from dkist_processing_common.models.fits_access import FitsAccessBase
+from dkist_processing_common.models.flower_pot import ListStem
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.flower_pot import Thorn
 from dkist_processing_common.models.tags import StemName
 from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.parsers.lookup_bud import TimeLookupBud
 from dkist_processing_common.parsers.single_value_single_key_flower import (
     SingleValueSingleKeyFlower,
 )
 from dkist_processing_common.parsers.unique_bud import UniqueBud
 from dkist_processing_common.tasks.parse_l0_input_data import ParseL0InputDataBase
+from dkist_processing_common.tasks.parse_l0_input_data import default_constant_bud_factory
 
 
 class VispHeaders(Spec122Dataset):
@@ -84,25 +91,44 @@ class VispHeaders(Spec122Dataset):
         return self.index % self.num_mod
 
 
+class ViSPMetadataKey(StrEnum):
+    num_mod = "VISP_010"
+    modstate = "VISP_011"
+    ip_task_type = "DKIST004"
+
+
 class ViSPFitsAccess(FitsAccessBase):
     def __init__(self, hdu, name, auto_squeeze=False):
         super().__init__(hdu, name, auto_squeeze=auto_squeeze)
-        self.num_mod: int = self.header["VISP_010"]
-        self.modstate: int = self.header["VISP_011"]
-        self.ip_task_type: str = self.header["DKIST004"]
+        self.num_mod: int = self.header[ViSPMetadataKey.num_mod]
+        self.modstate: int = self.header[ViSPMetadataKey.modstate]
+        self.ip_task_type: str = self.header[ViSPMetadataKey.ip_task_type]
         self.name = name
 
 
 @pytest.fixture(scope="function")
 def visp_flowers():
     return [
-        SingleValueSingleKeyFlower(tag_stem_name=StemName.modstate.value, metadata_key="modstate")
+        SingleValueSingleKeyFlower(
+            tag_stem_name=StemName.modstate, metadata_key=ViSPMetadataKey.modstate
+        )
     ]
 
 
 @pytest.fixture(scope="function")
 def visp_buds():
-    return [UniqueBud(constant_name=BudName.num_modstates.value, metadata_key="num_mod")]
+    return [UniqueBud(constant_name=BudName.num_modstates, metadata_key=ViSPMetadataKey.num_mod)]
+
+
+@pytest.fixture(scope="function")
+def visp_lookup_buds():
+    return [
+        TimeLookupBud(
+            constant_name="LOOKUP_BUD",
+            key_metadata_key=ViSPMetadataKey.num_mod,
+            value_metadata_key=ViSPMetadataKey.modstate,
+        )
+    ]
 
 
 @pytest.fixture(scope="function")
@@ -132,7 +158,27 @@ def empty_buds():
         def getter(self, key):
             pass  # We'll never get here because we spilled the dirt
 
-    return [EmptyBud()]
+    class EmptyListBud(ListStem):
+        def __init__(self):
+            super().__init__(stem_name="EMPTY_LIST_BUD")
+
+        def setter(self, value):
+            return SpilledDirt
+
+        def getter(self):
+            pass
+
+    class EmptySetBud(SetStem):
+        def __init__(self):
+            super().__init__(stem_name="EMPTY_SET_BUD")
+
+        def setter(self, value):
+            return SpilledDirt
+
+        def getter(self):
+            pass
+
+    return [EmptyBud(), EmptyListBud(), EmptySetBud()]
 
 
 @pytest.fixture()
@@ -149,8 +195,17 @@ def picky_buds():
 
 @pytest.fixture(scope="function")
 def parse_inputs_task(
-    tmp_path, visp_flowers, visp_buds, empty_flowers, empty_buds, picky_buds, recipe_run_id
+    tmp_path,
+    visp_flowers,
+    visp_buds,
+    visp_lookup_buds,
+    empty_flowers,
+    empty_buds,
+    picky_buds,
+    recipe_run_id,
 ):
+    """Override parse task class and make data for testing."""
+
     class TaskClass(ParseL0InputDataBase):
         @property
         def tag_flowers(self):
@@ -158,7 +213,7 @@ def parse_inputs_task(
 
         @property
         def constant_buds(self):
-            return visp_buds + empty_buds + picky_buds
+            return visp_buds + visp_lookup_buds + empty_buds + picky_buds
 
         @property
         def fits_parsing_class(self):
@@ -198,6 +253,8 @@ def parse_inputs_task(
 
 @pytest.fixture()
 def visp_parse_inputs_task(tmp_path, visp_flowers, visp_buds, recipe_run_id):
+    """Extend parse task class, but don't make data for testing."""
+
     class TaskClass(ParseL0InputDataBase):
         @property
         def tag_flowers(self):
@@ -231,12 +288,15 @@ def test_make_flowerpots(parse_inputs_task):
     tag_pot, constant_pot = parse_inputs_task.make_flower_pots()
 
     assert len(tag_pot.stems) == 2
-    assert len(constant_pot.stems) == 3
-    assert tag_pot.stems[0].stem_name == StemName.modstate.value
+    assert len(constant_pot.stems) == 6
+    assert tag_pot.stems[0].stem_name == StemName.modstate
     assert tag_pot.stems[1].stem_name == "EMPTY_FLOWER"
-    assert constant_pot.stems[0].stem_name == BudName.num_modstates.value
-    assert constant_pot.stems[1].stem_name == "EMPTY_BUD"
-    assert constant_pot.stems[2].stem_name == "PICKY_BUD"
+    assert constant_pot.stems[0].stem_name == BudName.num_modstates
+    assert constant_pot.stems[1].stem_name == "LOOKUP_BUD"
+    assert constant_pot.stems[2].stem_name == "EMPTY_BUD"
+    assert constant_pot.stems[3].stem_name == "EMPTY_LIST_BUD"
+    assert constant_pot.stems[4].stem_name == "EMPTY_SET_BUD"
+    assert constant_pot.stems[5].stem_name == "PICKY_BUD"
 
 
 def test_subclass_flowers(visp_parse_inputs_task, max_cs_step_time_sec):
@@ -248,36 +308,59 @@ def test_subclass_flowers(visp_parse_inputs_task, max_cs_step_time_sec):
     tag_pot, constant_pot = visp_parse_inputs_task.make_flower_pots()
 
     assert len(tag_pot.stems) == 1
-    assert len(constant_pot.stems) == 12
-    assert sorted([f.stem_name for f in tag_pot.stems]) == sorted([StemName.modstate.value])
-    assert sorted([f.stem_name for f in constant_pot.stems]) == sorted(
-        [
-            BudName.instrument.value,
-            BudName.num_modstates.value,
-            BudName.proposal_id.value,
-            BudName.contributing_proposal_ids.value,
-            BudName.average_cadence.value,
-            BudName.maximum_cadence.value,
-            BudName.minimum_cadence.value,
-            BudName.variance_cadence.value,
-            BudName.dark_exposure_times.value,
-            BudName.dark_readout_exp_times.value,
-            BudName.experiment_id.value,
-            BudName.contributing_experiment_ids.value,
-        ]
-    )
+    assert len(constant_pot.stems) == 61
+    all_flower_names = [StemName.modstate]
+    assert sorted([f.stem_name for f in tag_pot.stems]) == sorted(all_flower_names)
+    all_bud_names = [b.stem_name for b in default_constant_bud_factory()] + [BudName.num_modstates]
+    assert sorted([f.stem_name for f in constant_pot.stems]) == sorted(all_bud_names)
+
+
+def test_dataset_extra_bud_factory(visp_parse_inputs_task, max_cs_step_time_sec):
+    """
+    Given: ParseInputData child class with custom stems
+    When: Making the constant pot
+    Then: The multi-task dataset extra buds are created
+    """
+    _, constant_pot = visp_parse_inputs_task.make_flower_pots()
+    stem_names = [f.stem_name.value for f in constant_pot.stems]
+    bud_name_base = [
+        "DATE_BEGIN",
+        "OBSERVING_PROGRAM_EXECUTION_IDS",
+        "NUM_RAW_FRAMES_PER_FPA",
+        "TELESCOPE_TRACKING_MODE",
+        "COUDE_TABLE_TRACKING_MODE",
+        "TELESCOPE_SCANNING_MODE",
+        "AVERAGE_LIGHT_LEVEL",
+        "AVERAGE_TELESCOPE_ELEVATION",
+        "AVERAGE_COUDE_TABLE_ANGLE",
+        "AVERAGE_TELESCOPE_AZIMUTH",
+        "GOS_LEVEL3_STATUS",
+        "GOS_LEVEL3_LAMP_STATUS",
+        "GOS_POLARIZER_STATUS",
+        "GOS_POLARIZER_ANGLE",
+        "GOS_RETARDER_STATUS",
+        "GOS_RETARDER_ANGLE",
+        "GOS_LEVEL0_STATUS",
+    ]
+    for base in bud_name_base:
+        assert "SOLAR_GAIN_" + base in stem_names
+        # telescope mode keys are not constant for dark frames
+        assert ("DARK_" + base in stem_names) ^ ("MODE" in base)
+        # gos keys are not constant for polcal frames
+        assert ("POLCAL_" + base in stem_names) ^ ("GOS" in base)
 
 
 def test_constants_correct(parse_inputs_task):
     """
     Given: ParseInputData task with a populated constant FlowerPot
     When: Updating pipeline constants
-    Then: Pipeline constants are correctly populated
+    Then: A pipeline constant is correctly populated and the values return correctly
     """
     _, constant_pot = parse_inputs_task.make_flower_pots()
     parse_inputs_task.update_constants(constant_pot)
     assert dict(parse_inputs_task.constants._db_dict) == {
         BudName.num_modstates.value: parse_inputs_task._num_mod,
+        "LOOKUP_BUD": {str(parse_inputs_task._num_mod): [0, 1]},
     }
 
 

dkist_processing_common/tests/test_publish_catalog_messages.py

@@ -4,13 +4,12 @@ import pytest
 
 from dkist_processing_common.models.message import CreateQualityReportMessage
 from dkist_processing_common.tasks.l1_output_data import PublishCatalogAndQualityMessages
-from dkist_processing_common.tests.conftest import FakeGQLClient
 
 
 @pytest.fixture
-def publish_catalog_and_quality_messages_task(recipe_run_id, mocker):
+def publish_catalog_and_quality_messages_task(recipe_run_id, mocker, fake_gql_client):
     mocker.patch(
-        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
     )
     with PublishCatalogAndQualityMessages(
         recipe_run_id=recipe_run_id,
@@ -58,24 +57,3 @@ def test_object_messages(publish_catalog_and_quality_messages_task):
         assert message.body.conversationId == str(task.recipe_run_id)
         assert message.body.objectType == object_type
         assert message.body.groupId == task.constants.dataset_id
-
-
-def test_quality_report_message(publish_catalog_and_quality_messages_task):
-    """
-    :Given: a PublishCatalogAndQualityMessages task
-    :When: creating quality report message
-    :Then: the attributes are correctly populated
-    """
-    # Given
-    task, proposal_id = publish_catalog_and_quality_messages_task
-    # When
-    message = task.quality_report_message
-    # Then
-    assert isinstance(message, CreateQualityReportMessage)
-    assert message.body.bucket == task.destination_bucket
-    # objectName exists and can be evaluated as a valid path
-    assert message.body.objectName
-    _ = Path(message.body.objectName)
-    assert message.body.datasetId == task.constants.dataset_id
-    assert message.body.conversationId == str(task.recipe_run_id)
-    assert message.body.incrementDatasetCatalogReceiptCount is True

dkist_processing_common/tests/test_quality.py

@@ -1,4 +1,5 @@
 """Tests for the quality tasks."""
+
 import json
 from typing import Iterable