dkist-processing-common 12.0.0rc5__py3-none-any.whl → 12.2.0__py3-none-any.whl

dkist_processing_common/tests/test_construct_dataset_extras.py

@@ -0,0 +1,224 @@
+from dataclasses import asdict
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+import pytest
+from astropy.io import fits
+
+from dkist_processing_common._util.scratch import WorkflowFileSystem
+from dkist_processing_common.codecs.fits import fits_array_decoder
+from dkist_processing_common.codecs.fits import fits_array_encoder
+from dkist_processing_common.models.extras import DatasetExtraHeaderSection
+from dkist_processing_common.models.extras import DatasetExtraType
+from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.models.task_name import TaskName
+from dkist_processing_common.tasks.write_extra import WriteL1DatasetExtras
+from dkist_processing_common.tests.mock_metadata_store import RecipeRunResponseMapping
+from dkist_processing_common.tests.mock_metadata_store import fake_gql_client_factory
+from dkist_processing_common.tests.mock_metadata_store import make_default_recipe_run_response
+
+
+@dataclass
+class FakeConstantDb:
+    INSTRUMENT: str = "VBI"
+    DATASET_ID: str = "DATASETID"
+    AVERAGE_CADENCE: float = 10.0
+    MINIMUM_CADENCE: float = 10.0
+    MAXIMUM_CADENCE: float = 10.0
+    VARIANCE_CADENCE: float = 0.0
+    STOKES_PARAMS: tuple = ("I", "Q", "U", "V")
+    PROPOSAL_ID: str = "PROPID1"
+    EXPERIMENT_ID: str = "EXPERID1"
+    CAMERA_ID: str = "CAMERA1"
+    CAMERA_NAME: str = "Camera One"
+    CAMERA_BIT_DEPTH: int = 16
+    HARDWARE_BINNING_X: int = 1
+    HARDWARE_BINNING_Y: int = 1
+    SOFTWARE_BINNING_X: int = 1
+    SOFTWARE_BINNING_Y: int = 1
+    HLS_VERSION: str = "1.8"
+    WAVELENGTH: float = 854.2
+    # Dark
+    DARK_OBSERVING_PROGRAM_EXECUTION_IDS: tuple = ("OP1", "OP2", "OP3")
+    DARK_DATE_BEGIN: str = "2023-01-01T00:00:00"
+    DARK_DATE_END: str = "2023-01-01T01:00:00"
+    DARK_TELESCOPE_TRACKING_MODE: str = "None"
+    DARK_COUDE_TABLE_TRACKING_MODE: str = "fixed coude table angle"
+    DARK_TELESCOPE_SCANNING_MODE: str = "None"
+    DARK_AVERAGE_LIGHT_LEVEL: float = 5.0
+    DARK_AVERAGE_TELESCOPE_ELEVATION: float = 45.0
+    DARK_AVERAGE_COUDE_TABLE_ANGLE: float = 2.0
+    DARK_AVERAGE_TELESCOPE_AZIMUTH: float = 180.0
+    DARK_GOS_LEVEL3_STATUS: str = "clear"
+    DARK_GOS_LEVEL3_LAMP_STATUS: str = "off"
+    DARK_GOS_POLARIZER_STATUS: str = "clear"
+    DARK_GOS_POLARIZER_ANGLE: str = "0.0"
+    DARK_GOS_RETARDER_STATUS: str = "clear"
+    DARK_GOS_RETARDER_ANGLE: str = "0.0"
+    DARK_GOS_LEVEL0_STATUS: str = "DarkShutter"
+    # Solar Gain
+    SOLAR_GAIN_OBSERVING_PROGRAM_EXECUTION_IDS: tuple = ("OP1", "OP2", "OP3")
+    SOLAR_GAIN_DATE_BEGIN: str = "2023-01-01T00:00:00"
+    SOLAR_GAIN_DATE_END: str = "2023-01-01T01:00:00"
+    SOLAR_GAIN_NUM_RAW_FRAMES_PER_FPA: int = 1
+    SOLAR_GAIN_TELESCOPE_TRACKING_MODE: str = "None"
+    SOLAR_GAIN_COUDE_TABLE_TRACKING_MODE: str = "fixed coude table angle"
+    SOLAR_GAIN_TELESCOPE_SCANNING_MODE: str = "None"
+    SOLAR_GAIN_AVERAGE_LIGHT_LEVEL: float = 5.0
+    SOLAR_GAIN_AVERAGE_TELESCOPE_ELEVATION: float = 45.0
+    SOLAR_GAIN_AVERAGE_COUDE_TABLE_ANGLE: float = 2.0
+    SOLAR_GAIN_AVERAGE_TELESCOPE_AZIMUTH: float = 180.0
+    SOLAR_GAIN_GOS_LEVEL3_STATUS: str = "clear"
+    SOLAR_GAIN_GOS_LEVEL3_LAMP_STATUS: str = "off"
+    SOLAR_GAIN_GOS_POLARIZER_STATUS: str = "clear"
+    SOLAR_GAIN_GOS_POLARIZER_ANGLE: str = "0.0"
+    SOLAR_GAIN_GOS_RETARDER_STATUS: str = "clear"
+    SOLAR_GAIN_GOS_RETARDER_ANGLE: str = "0.0"
+    SOLAR_GAIN_GOS_LEVEL0_STATUS: str = "DarkShutter"
+    # Polcal
+    POLCAL_OBSERVING_PROGRAM_EXECUTION_IDS: tuple = ("OP1", "OP2", "OP3")
+    POLCAL_DATE_BEGIN: str = "2023-01-01T00:00:00"
+    POLCAL_DATE_END: str = "2023-01-01T01:00:00"
+    POLCAL_NUM_RAW_FRAMES_PER_FPA: int = 1
+    POLCAL_TELESCOPE_TRACKING_MODE: str = "None"
+    POLCAL_COUDE_TABLE_TRACKING_MODE: str = "fixed coude table angle"
+    POLCAL_TELESCOPE_SCANNING_MODE: str = "None"
+    POLCAL_AVERAGE_LIGHT_LEVEL: float = 5.0
+    POLCAL_AVERAGE_TELESCOPE_ELEVATION: float = 45.0
+    POLCAL_AVERAGE_COUDE_TABLE_ANGLE: float = 2.0
+    POLCAL_AVERAGE_TELESCOPE_AZIMUTH: float = 180.0
+
+
+class ConstructDatasetExtrasTest(WriteL1DatasetExtras):
+    def run(self):
+        # Make a dataset extra for each task type
+
+        for task_type in [
+            TaskName.dark,
+            TaskName.solar_gain,
+        ]:
+            filename = self.format_extra_filename(task_type, detail="BEAM1")
+            data = next(
+                self.read(
+                    tags=[Tag.task(task_type), Tag.intermediate()], decoder=fits_array_decoder
+                )
+            )
+            header = self.build_dataset_extra_header(
+                sections=[
+                    DatasetExtraHeaderSection.common,
+                    DatasetExtraHeaderSection.aggregate,
+                    DatasetExtraHeaderSection.iptask,
+                    DatasetExtraHeaderSection.gos,
+                ],
+                filename=filename,
+                task_type=task_type,
+                total_exposure=0.058,
+                readout_exposure=0.029,
+                extra_name=(
+                    DatasetExtraType.dark if task_type == "DARK" else DatasetExtraType.solar_gain
+                ),
+                end_time="2025-01-01T00:00:00",
+            )
+
+            self.assemble_and_write_dataset_extra(data=data, header=header, filename=filename)
+
+        task_type = TaskName.polcal
+        filename = self.format_extra_filename(task_type, detail="BEAM1")
+        data = next(
+            self.read(tags=[Tag.task(task_type), Tag.intermediate()], decoder=fits_array_decoder)
+        )
+        header = self.build_dataset_extra_header(
+            sections=[
+                DatasetExtraHeaderSection.common,
+                DatasetExtraHeaderSection.aggregate,
+                DatasetExtraHeaderSection.iptask,
+                DatasetExtraHeaderSection.gos,
+            ],
+            filename=filename,
+            task_type=task_type,
+            total_exposure=0.058,
+            readout_exposure=0.029,
+            extra_name=DatasetExtraType.demodulation_matrices,
+            end_time="2025-01-01T00:00:00",
+        )
+        self.assemble_and_write_dataset_extra(data=data, header=header, filename=filename)
+
+
+@pytest.fixture()
+def construct_dataset_extras_task(request, recipe_run_id, tmp_path):
+    with ConstructDatasetExtrasTest(
+        recipe_run_id=recipe_run_id,
+        workflow_name="workflow_name",
+        workflow_version="workflow_version",
+    ) as task:
+        task.scratch = WorkflowFileSystem(recipe_run_id=recipe_run_id, scratch_base_path=tmp_path)
+        # Write an intermediate product to be used as the source for each dataset extra
+        for task_type in [
+            TaskName.dark,
+            TaskName.solar_gain,
+            TaskName.polcal,
+        ]:
+            task.write(
+                data=np.random.random(size=(1, 128, 128)),
+                tags=[Tag.task(task_type), Tag.intermediate()],
+                encoder=fits_array_encoder,
+            )
+        task.constants._update(asdict(FakeConstantDb()))
+        yield task
+        task._purge()
+
+
+@pytest.fixture
+def fake_gql_client_default_configuration():
+    """Create GraphQL client Mock that returns result without recipe run configuration."""
+    recipe_run_response = make_default_recipe_run_response()
+    recipe_run_response.configuration = None
+    new_response_mapping = RecipeRunResponseMapping(response=recipe_run_response)
+    FakeGQLClientDefaultConfiguration = fake_gql_client_factory(
+        response_mapping_override=new_response_mapping
+    )
+
+    return FakeGQLClientDefaultConfiguration
+
+
+def test_construct_dataset_extras(
+    construct_dataset_extras_task, mocker, fake_gql_client_default_configuration
+):
+    """
+    Given: A ConstructDatasetExtras task with source data
+    When: Running the ConstructDatasetExtras task
+    Then: A dataset extra files are produced with expected header values
+    """
+    mocker.patch(
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
+        new=fake_gql_client_default_configuration,
+    )
+    task = construct_dataset_extras_task
+    task()
+    dataset_extra_files = list(task.read(tags=[Tag.output(), Tag.extra()]))
+    assert len(dataset_extra_files) == 3
+    for filename in dataset_extra_files:
+        split_filename = Path(filename).name.split("_")
+        assert split_filename[0] == "VBI"
+        assert split_filename[1] == task.constants.dataset_id
+        assert split_filename[-2] == "BEAM1"
+        assert split_filename[-1] == "1.fits"
+        hdul = fits.open(filename)
+        for i in range(1, len(hdul)):
+            assert isinstance(hdul[i], fits.CompImageHDU)
+            header = hdul[i].header
+            assert header["LINEWAV"] == 854.2
+            assert header["INSTRUME"] == "VBI"
+            assert header["ATAZIMUT"] == 180.0
+            assert header["FRAMEVOL"] is not None
+            assert header["IDSOBSID"] == 2
+            assert header["XPOSURE"] == 0.058
+            assert header["OBSPR_ID"] == "OP1"
+            assert header["EXTOBSID"] == "OP2,OP3"
+            assert header["EXTNAME"] in ["DARK", "SOLAR GAIN", "DEMODULATION MATRICES"]
+            if header["IPTASK"] == "POLCAL":
+                assert "POLANGLE" not in header
+            else:
+                assert header.get("POLANGLE") == "0.0"
+                assert header.get("RETANGLE") == "0.0"

dkist_processing_common/tests/test_flower_pot.py

@@ -4,11 +4,13 @@ from typing import Hashable
 import pytest
 
 from dkist_processing_common.models.flower_pot import FlowerPot
+from dkist_processing_common.models.flower_pot import ListStem
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.models.flower_pot import Stem
 
 
-@pytest.fixture()
+@pytest.fixture
 def simple_flower():
     class Flower(Stem):
         def setter(self, value: Any) -> Any:
@@ -22,7 +24,7 @@ def simple_flower():
     return Flower(stem_name="simple_flower")
 
 
-@pytest.fixture()
+@pytest.fixture
 def simple_flower_pot(simple_flower):
     flower_pot = FlowerPot()
     flower_pot.stems += [simple_flower]
@@ -30,16 +32,66 @@ def simple_flower_pot(simple_flower):
     return flower_pot
 
 
-@pytest.fixture()
+@pytest.fixture
 def simple_key_values():
     return {f"thing{i}": i for i in range(5)}
 
 
+@pytest.fixture
+def stem_bud():
+    class Bud(Stem):
+        def setter(self, value: int) -> int:
+            return value % 3
+
+        def getter(self, key: str) -> int:
+            return len(set(self.key_to_petal_dict.values()))
+
+    return Bud(stem_name="StemBud")
+
+
+@pytest.fixture
+def setstem_bud():
+    # Computes the same result as `stem_bud`
+    class SetBud(SetStem):
+        def setter(self, value: int) -> int:
+            return value % 3
+
+        def getter(self) -> int:
+            return len(self.value_set)
+
+    return SetBud(stem_name="SetStemBud")
+
+
+@pytest.fixture
+def liststem_bud():
+    # Highlights the difference between using a `set` and a `list` in these more efficient buds
+    class ListBud(ListStem):
+        def setter(self, value: int) -> int:
+            return value % 3
+
+        def getter(self) -> int:
+            return len(self.value_list)
+
+    return ListBud(stem_name="ListStemBud")
+
+
+@pytest.fixture
+def simple_bud_pot(stem_bud, setstem_bud, liststem_bud):
+    bud_pot = FlowerPot()
+    bud_pot.stems += [stem_bud, liststem_bud, setstem_bud]
+    return bud_pot
+
+
+@pytest.fixture
+def bud_key_values():
+    return {f"DOESN'T_MATTER_THAT'S_THE_POINT_{i}": i for i in [0, 1, 3, 4]}
+
+
 def test_simple_flower_pot(simple_flower_pot, simple_key_values):
     """
     Given: A FlowerPot with a simple Flower
-    When: Updating rock and flower with key: value pairs
-    Then: The rock and flower are correctly updated
+    When: Updating flower with key: value pairs
+    Then: The flower are correctly updated
     """
     assert len(simple_flower_pot) == 1
 
@@ -51,6 +103,8 @@ def test_simple_flower_pot(simple_flower_pot, simple_key_values):
 
     petals = sorted(list(flower.petals), key=lambda x: x.value)
     assert len(petals) == 2
+    assert flower.bud.value == petals[0].value
+    assert flower.bud.keys == petals[0].keys
     assert petals[0].value == 0
     assert petals[0].keys == ["thing0", "thing2", "thing4"]
     assert petals[1].value == 1
@@ -67,8 +121,12 @@ def test_cached_petal(simple_flower):
     value1 = 4
     simple_flower.update(key1, value1)
     assert len(simple_flower.petals) == 1
+
+    # Assert twice to hit the cache
     assert simple_flower.petals[0].value == value1 % 2  # % 2 because of simple_flower's `setter`
     assert simple_flower.petals[0].keys == [key1]
+    assert simple_flower.petals[0].value == value1 % 2
+    assert simple_flower.petals[0].keys == [key1]
 
     key2 = "thing2"
     value2 = 3
@@ -79,6 +137,10 @@ def test_cached_petal(simple_flower):
     assert sorted_petals[0].keys == [key1]
     assert sorted_petals[1].value == value2 % 2
     assert sorted_petals[1].keys == [key2]
+    assert sorted_petals[0].value == value1 % 2
+    assert sorted_petals[0].keys == [key1]
+    assert sorted_petals[1].value == value2 % 2
+    assert sorted_petals[1].keys == [key2]
 
 
 def test_spilled_dirt_flower(simple_flower):
@@ -103,3 +165,83 @@ def test_unhashable_dirt(simple_flower_pot):
     value = "never gonna get here"
     with pytest.raises(TypeError):
         simple_flower_pot.add_dirt(unhashable_key, value)
+
+
+def test_buds(simple_bud_pot, bud_key_values):
+    """
+    Given: A Flower pot with two Buds that compute the same thing; one a `Stem` and one a `SetStem`, and a `ListStem` bud
+    When: Updating the pot with key: value pairs
+    Then: The computed buds are correct and the `Stem` and `SetStem` buds match
+    """
+    assert len(simple_bud_pot) == 3
+
+    assert simple_bud_pot[0].stem_name == "StemBud"
+    assert simple_bud_pot[1].stem_name == "ListStemBud"
+    assert simple_bud_pot[2].stem_name == "SetStemBud"
+
+    for k, m in bud_key_values.items():
+        simple_bud_pot.add_dirt(k, m)
+
+    assert simple_bud_pot[0].bud.value == 2
+    assert simple_bud_pot[1].bud.value == 4
+    assert simple_bud_pot[2].bud.value == simple_bud_pot[0].bud.value
+
+    assert len(simple_bud_pot[0].petals) == 1
+    assert simple_bud_pot[0].petals[0].value == 2
+    assert simple_bud_pot[0].petals[0].keys == [
+        f"DOESN'T_MATTER_THAT'S_THE_POINT_{i}" for i in [0, 1, 3, 4]
+    ]
+
+    with pytest.raises(
+        AttributeError,
+        match="ListBud subclasses ListStem and therefore does not define the `petals` property",
+    ):
+        _ = simple_bud_pot[1].petals
+
+    with pytest.raises(
+        AttributeError,
+        match="SetBud subclasses SetStem and therefore does not define the `petals` property",
+    ):
+        _ = simple_bud_pot[2].petals
+
+
+def test_liststem_cached_bud(liststem_bud):
+    """
+    Given: A `ListStem` instance and some different input values
+    When: Computing the `bud` property after each value is ingested
+    Then: The `bud` value correctly updates based on the state of the `ListStem` object
+    """
+    key = "Who cares"
+    value1 = 3
+    liststem_bud.update(key, value1)
+
+    # Assert twice so we hit the cache
+    assert liststem_bud.bud.value == 1
+    assert liststem_bud.bud.value == 1
+
+    value2 = 1
+    liststem_bud.update(key, value2)
+
+    assert liststem_bud.bud.value == 2
+    assert liststem_bud.bud.value == 2
+
+
+def test_setstem_cached_bud(setstem_bud):
+    """
+    Given: A `SetStem` instance and some different input values
+    When: Computing the `bud` property after each value is ingested
+    Then: The `bud` value correctly updates based on the state of the `SetStem` object
+    """
+    key = "Who cares"
+    value1 = 3
+    setstem_bud.update(key, value1)
+
+    # Assert twice so we hit the cache
+    assert setstem_bud.bud.value == 1
+    assert setstem_bud.bud.value == 1
+
+    value2 = 1
+    setstem_bud.update(key, value2)
+
+    assert setstem_bud.bud.value == 2
+    assert setstem_bud.bud.value == 2

dkist_processing_common/tests/test_output_data_base.py

@@ -57,7 +57,11 @@ def transfer_data_task(recipe_run_id, tmp_path, mocker, fake_gql_client):
         unwanted_file_obj = uuid4().hex.encode("utf8")
         task.write(unwanted_file_obj, tags=[Tag.frame()])
 
-        yield task, output_file_obj
+        # Write a dataset extra
+        extra_file_obj = uuid4().hex.encode("utf8")
+        task.write(extra_file_obj, tags=[Tag.output(), Tag.extra()])
+
+        yield task, output_file_obj, extra_file_obj
         task._purge()
 
 
@@ -81,7 +85,7 @@ def test_build_output_frame_transfer_list(transfer_data_task):
     When: Building a transfer list of all OUTPUT frames
     Then: All OUTPUT frames are listed and no non-OUTPUT frames are listed
     """
-    task, output_file_obj = transfer_data_task
+    task, output_file_obj, _ = transfer_data_task
 
     transfer_list = task.build_output_frame_transfer_list()
 
@@ -89,3 +93,21 @@ def test_build_output_frame_transfer_list(transfer_data_task):
     transfer_item = transfer_list[0]
     with transfer_item.source_path.open(mode="rb") as f:
         assert output_file_obj == f.read()
+
+
+def test_build_dataset_extra_transfer_list(transfer_data_task):
+    """
+    Given: A task based on TransferDataBase with some files, some of which are EXTRA_OUTPUT
+    When: Building a transfer list of all EXTRA_OUTPUT frames
+    Then: All EXTRA_OUTPUT frames are listed and no non-EXTRA_OUTPUT frames are listed
+    """
+    task, _, extra_file_obj = transfer_data_task
+
+    transfer_list = task.build_dataset_extra_transfer_list()
+
+    assert len(transfer_list) == 1
+    transfer_item = transfer_list[0]
+    assert "/extra/" not in str(transfer_item.source_path)
+    assert "/extra/" in str(transfer_item.destination_path)
+    with transfer_item.source_path.open(mode="rb") as f:
+        assert extra_file_obj == f.read()

dkist_processing_common/tests/test_parse_l0_input_data.py

@@ -12,6 +12,8 @@ from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
 from dkist_processing_common.models.constants import BudName
 from dkist_processing_common.models.fits_access import FitsAccessBase
+from dkist_processing_common.models.flower_pot import ListStem
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.flower_pot import Thorn
@@ -156,7 +158,27 @@ def empty_buds():
         def getter(self, key):
             pass  # We'll never get here because we spilled the dirt
 
-    return [EmptyBud()]
+    class EmptyListBud(ListStem):
+        def __init__(self):
+            super().__init__(stem_name="EMPTY_LIST_BUD")
+
+        def setter(self, value):
+            return SpilledDirt
+
+        def getter(self):
+            pass
+
+    class EmptySetBud(SetStem):
+        def __init__(self):
+            super().__init__(stem_name="EMPTY_SET_BUD")
+
+        def setter(self, value):
+            return SpilledDirt
+
+        def getter(self):
+            pass
+
+    return [EmptyBud(), EmptyListBud(), EmptySetBud()]
 
 
 @pytest.fixture()
@@ -266,13 +288,15 @@ def test_make_flowerpots(parse_inputs_task):
     tag_pot, constant_pot = parse_inputs_task.make_flower_pots()
 
     assert len(tag_pot.stems) == 2
-    assert len(constant_pot.stems) == 4
+    assert len(constant_pot.stems) == 6
     assert tag_pot.stems[0].stem_name == StemName.modstate
     assert tag_pot.stems[1].stem_name == "EMPTY_FLOWER"
     assert constant_pot.stems[0].stem_name == BudName.num_modstates
     assert constant_pot.stems[1].stem_name == "LOOKUP_BUD"
     assert constant_pot.stems[2].stem_name == "EMPTY_BUD"
-    assert constant_pot.stems[3].stem_name == "PICKY_BUD"
+    assert constant_pot.stems[3].stem_name == "EMPTY_LIST_BUD"
+    assert constant_pot.stems[4].stem_name == "EMPTY_SET_BUD"
+    assert constant_pot.stems[5].stem_name == "PICKY_BUD"
 
 
 def test_subclass_flowers(visp_parse_inputs_task, max_cs_step_time_sec):
@@ -336,7 +360,7 @@ def test_constants_correct(parse_inputs_task):
     parse_inputs_task.update_constants(constant_pot)
     assert dict(parse_inputs_task.constants._db_dict) == {
         BudName.num_modstates.value: parse_inputs_task._num_mod,
-        "LOOKUP_BUD": [[parse_inputs_task._num_mod, [0, 1]]],
+        "LOOKUP_BUD": {str(parse_inputs_task._num_mod): [0, 1]},
     }