dkist-processing-common 12.0.0rc7__py3-none-any.whl → 12.1.0__py3-none-any.whl

dkist_processing_common/parsers/time.py

@@ -4,15 +4,15 @@ from datetime import datetime
 from datetime import timezone
 from enum import StrEnum
 from typing import Callable
-from typing import Hashable
 from typing import Type
 
 import numpy as np
 
 from dkist_processing_common.models.constants import BudName
 from dkist_processing_common.models.fits_access import MetadataKey
+from dkist_processing_common.models.flower_pot import ListStem
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
-from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.tags import EXP_TIME_ROUND_DIGITS
 from dkist_processing_common.models.tags import StemName
 from dkist_processing_common.models.task_name import TaskName
@@ -35,7 +35,7 @@ class ObsIpStartTimeBud(TaskUniqueBud):
         )
 
 
-class TaskDatetimeBudBase(Stem):
+class TaskDatetimeBudBase(ListStem):
     """
     Base class for making datetime-related buds.
 
@@ -59,8 +59,6 @@ class TaskDatetimeBudBase(Stem):
         The function used to convert a header into an IP task type
     """
 
-    key_to_petal_dict: dict[str, float]
-
     def __init__(
         self,
         stem_name: str,
@@ -101,20 +99,15 @@ class TaskDatetimeBudBase(Stem):
 
         return SpilledDirt
 
-    def getter(self, key: Hashable) -> tuple[float, ...]:
+    def getter(self) -> tuple[float, ...]:
         """
         Return a tuple of sorted times in unix seconds.
 
-        Parameters
-        ----------
-        key
-            The input key
-
         Returns
         -------
         A tuple that is sorted times in unix seconds
         """
-        return tuple(sorted(list(self.key_to_petal_dict.values())))
+        return tuple(sorted(self.value_list))
 
 
 class CadenceBudBase(TaskDatetimeBudBase):
@@ -134,20 +127,15 @@ class AverageCadenceBud(CadenceBudBase):
     def __init__(self):
         super().__init__(constant_name=BudName.average_cadence)
 
-    def getter(self, key) -> np.float64:
+    def getter(self) -> np.float64:
         """
         Return the mean cadence between frames.
 
-        Parameters
-        ----------
-        key
-            The input key
-
         Returns
         -------
         The mean value of the cadences of the input frames
         """
-        return np.mean(np.diff(super().getter(key)))
+        return np.mean(np.diff(super().getter()))
 
 
 class MaximumCadenceBud(CadenceBudBase):
@@ -156,20 +144,15 @@ class MaximumCadenceBud(CadenceBudBase):
     def __init__(self):
         super().__init__(constant_name=BudName.maximum_cadence)
 
-    def getter(self, key) -> np.float64:
+    def getter(self) -> np.float64:
         """
         Return the maximum cadence between frames.
 
-        Parameters
-        ----------
-        key
-            The input key
-
         Returns
         -------
         The maximum cadence between frames
         """
-        return np.max(np.diff(super().getter(key)))
+        return np.max(np.diff(super().getter()))
 
 
 class MinimumCadenceBud(CadenceBudBase):
@@ -178,20 +161,15 @@ class MinimumCadenceBud(CadenceBudBase):
     def __init__(self):
         super().__init__(constant_name=BudName.minimum_cadence)
 
-    def getter(self, key) -> np.float64:
+    def getter(self) -> np.float64:
         """
         Return the minimum cadence between frames.
 
-        Parameters
-        ----------
-        key
-            The input key
-
         Returns
         -------
         The minimum cadence between frames
         """
-        return np.min(np.diff(super().getter(key)))
+        return np.min(np.diff(super().getter()))
 
 
 class VarianceCadenceBud(CadenceBudBase):
@@ -200,19 +178,15 @@ class VarianceCadenceBud(CadenceBudBase):
     def __init__(self):
         super().__init__(constant_name=BudName.variance_cadence)
 
-    def getter(self, key) -> np.float64:
+    def getter(self) -> np.float64:
         """
         Return the cadence variance between frames.
 
-        Parameters
-        ----------
-        key
-            The input key
         Returns
         -------
         Return the variance of the cadences over the input frames
         """
-        return np.var(np.diff(super().getter(key)))
+        return np.var(np.diff(super().getter()))
 
 
 class TaskDateBeginBud(TaskDatetimeBudBase):
@@ -231,19 +205,16 @@ class TaskDateBeginBud(TaskDatetimeBudBase):
             task_type_parsing_function=task_type_parsing_function,
         )
 
-    def getter(self, key) -> str:
+    def getter(self) -> str:
         """
         Return the earliest date begin for the ip task type converted from unix seconds to datetime string.
 
-        Parameters
-        ----------
-        key
-            The input key
         Returns
         -------
         Return the minimum date begin as a datetime string
         """
-        min_time = super().getter(key)[0]
+        # super().getter() returns a sorted list
+        min_time = super().getter()[0]
         min_time_dt = datetime.fromtimestamp(min_time, tz=timezone.utc)
         return min_time_dt.strftime("%Y-%m-%dT%H:%M:%S.%f")
 
@@ -286,7 +257,7 @@ class ReadoutExpTimeFlower(RoundTimeFlowerBase):
         )
 
 
-class TaskRoundTimeBudBase(Stem):
+class TaskRoundTimeBudBase(SetStem):
     """
     Base class for making buds that need a set of rounded times for computing for specific task types.
 
@@ -310,8 +281,6 @@ class TaskRoundTimeBudBase(Stem):
         The function used to convert a header into an IP task type
     """
 
-    key_to_petal_dict: dict[str, float]
-
     def __init__(
         self,
         stem_name: str,
@@ -349,20 +318,15 @@ class TaskRoundTimeBudBase(Stem):
 
         return SpilledDirt
 
-    def getter(self, key: Hashable) -> tuple[float, ...]:
+    def getter(self) -> tuple[float, ...]:
         """
         Return a tuple of the sorted unique values found.
 
-        Parameters
-        ----------
-        key
-            The input key
-
         Returns
         -------
         A tuple that is the sorted set of unique times
         """
-        return tuple(sorted(set(self.key_to_petal_dict.values())))
+        return tuple(sorted(self.value_set))
 
 
 class TaskExposureTimesBud(TaskRoundTimeBudBase):

dkist_processing_common/parsers/unique_bud.py

@@ -3,15 +3,15 @@
 from enum import StrEnum
 from typing import Callable
 
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
-from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.parsers.task import passthrough_header_ip_task
 
 
-class UniqueBud(Stem):
+class UniqueBud(SetStem):
     """
-    Pre-made flower that reads a single header key from all files and raises a ValueError if it is not unique.
+    Pre-made `SetStem` that reads a single header key from all files and raises a ValueError if it is not unique.
 
     Parameters
     ----------
@@ -46,24 +46,17 @@ class UniqueBud(Stem):
         """
         return getattr(fits_obj, self.metadata_key)
 
-    def getter(self, key):
+    def getter(self):
         """
         Get the value for this key and raise an error if it is not unique.
 
-        Parameters
-        ----------
-        key
-            The input key
         Returns
         -------
         The value associated with this input key
         """
-        value_set = set(self.key_to_petal_dict.values())
-        if len(value_set) > 1:
-            raise ValueError(
-                f"Multiple {self.stem_name} values found for key {key}. Values: {value_set}"
-            )
-        return value_set.pop()
+        if len(self.value_set) > 1:
+            raise ValueError(f"Multiple {self.stem_name} values found! Values: {self.value_set}")
+        return self.value_set.pop()
 
 
 class TaskUniqueBud(UniqueBud):

dkist_processing_common/tasks/parse_l0_input_data.py

@@ -397,6 +397,8 @@ class ParseDataBase(WorkflowTaskBase, ABC):
         return self.read(
             tags=self.tags_for_input_frames,
             decoder=fits_access_decoder,
+            checksum=False,
+            disable_image_compression=True,
             fits_access_class=self.fits_parsing_class,
         )
 
@@ -419,8 +421,8 @@ class ParseDataBase(WorkflowTaskBase, ABC):
         """
         for stem in constant_pot:
             with self.telemetry_span(f"Setting value of constant {stem.stem_name}"):
-                if len(stem.petals) == 0:
-                    # There are no petals so nothing to do
+                if not stem.can_be_picked:
+                    # Nothing to do
                     continue
                 if stem.bud.value is Thorn:
                     # Must've been a picky bud that passed. We don't want to pick it because it has no value

dkist_processing_common/tasks/transfer_input_data.py

@@ -78,6 +78,7 @@ class TransferL0Data(WorkflowTaskBase, GlobusMixin):
                 self.tag(obj_path, tags=obj.tag)
             else:
                 self.tag(obj_path, tags=[Tag.input(), Tag.frame()])
+        logger.info(f"Tagged {len(input_dataset_objects)} input dataset objects in scratch")
 
     def run(self) -> None:
         """Execute the data transfer."""

dkist_processing_common/tests/test_codecs.py

@@ -183,7 +183,7 @@ def primary_hdu_list(ndarray_object, fits_header) -> HDUList:
 
 @pytest.fixture
 def path_to_primary_fits_file(primary_hdu_list, tmp_file) -> Path:
-    primary_hdu_list.writeto(tmp_file)
+    primary_hdu_list.writeto(tmp_file, checksum=True)
     return tmp_file
 
 
@@ -210,7 +210,7 @@ def compressed_hdu_list(ndarray_object, fits_header) -> HDUList:
 
 @pytest.fixture
 def path_to_compressed_fits_file(compressed_hdu_list, tmp_file) -> Path:
-    compressed_hdu_list.writeto(tmp_file)
+    compressed_hdu_list.writeto(tmp_file, checksum=True)
     return tmp_file
 
 
@@ -472,16 +472,27 @@ def test_bytesio_decoder(bytesIO_object, path_to_bytesIO):
         pytest.param("path_to_compressed_fits_file", id="compressed"),
     ],
 )
-def test_fits_hdu_decoder(path_fixture_name, ndarray_object, fits_header, request):
+@pytest.mark.parametrize(
+    "checksum", [pytest.param(True, id="checksum"), pytest.param(False, id="no_checksum")]
+)
+@pytest.mark.parametrize(
+    "decompress", [pytest.param(True, id="decompress"), pytest.param(False, id="no_decompress")]
+)
+def test_fits_hdu_decoder(
+    path_fixture_name, ndarray_object, fits_header, request, checksum, decompress
+):
     """
     Given: Path to a FITS file
     When: Decoding the path with the fits_hdu_decoder
     Then: The correct data are returned
     """
     file_path = request.getfixturevalue(path_fixture_name)
-    hdu = fits_hdu_decoder(file_path)
+    hdu = fits_hdu_decoder(file_path, checksum=checksum, disable_image_compression=not decompress)
 
-    assert np.array_equal(hdu.data, ndarray_object)
+    if "compressed" in path_fixture_name and not decompress:
+        assert not np.array_equal(hdu.data, ndarray_object)
+    else:
+        assert np.array_equal(hdu.data, ndarray_object)
     assert hdu.header["foo"] == fits_header["foo"]
 
 
@@ -492,7 +503,15 @@ def test_fits_hdu_decoder(path_fixture_name, ndarray_object, fits_header, reques
         pytest.param("path_to_compressed_fits_file", id="compressed"),
     ],
 )
-def test_fits_access_decoder(path_fixture_name, ndarray_object, fits_header, request):
+@pytest.mark.parametrize(
+    "checksum", [pytest.param(True, id="checksum"), pytest.param(False, id="no_checksum")]
+)
+@pytest.mark.parametrize(
+    "decompress", [pytest.param(True, id="decompress"), pytest.param(False, id="no_decompress")]
+)
+def test_fits_access_decoder(
+    path_fixture_name, ndarray_object, fits_header, request, checksum, decompress
+):
     """
     Given: Path to a FITS file
     When: Decoding the path with the fits_access_decoder
@@ -500,20 +519,47 @@ def test_fits_access_decoder(path_fixture_name, ndarray_object, fits_header, req
     """
     file_path = request.getfixturevalue(path_fixture_name)
 
-    fits_obj = fits_access_decoder(file_path, fits_access_class=DummyFitsAccess)
+    fits_obj = fits_access_decoder(
+        file_path,
+        fits_access_class=DummyFitsAccess,
+        checksum=checksum,
+        disable_image_compression=not decompress,
+    )
     assert fits_obj.name == str(file_path)
-    assert np.array_equal(fits_obj.data, ndarray_object)
     assert fits_obj.foo == fits_header["foo"]
+    if "compressed" in path_fixture_name and not decompress:
+        assert not np.array_equal(fits_obj.data, ndarray_object)
+    else:
+        assert np.array_equal(fits_obj.data, ndarray_object)
 
 
-def test_fits_array_decoder(path_to_primary_fits_file, ndarray_object):
+@pytest.mark.parametrize(
+    "path_fixture_name",
+    [
+        pytest.param("path_to_primary_fits_file", id="uncompressed"),
+        pytest.param("path_to_compressed_fits_file", id="compressed"),
+    ],
+)
+@pytest.mark.parametrize(
+    "checksum", [pytest.param(True, id="checksum"), pytest.param(False, id="no_checksum")]
+)
+@pytest.mark.parametrize(
+    "decompress", [pytest.param(True, id="decompress"), pytest.param(False, id="no_decompress")]
+)
+def test_fits_array_decoder(path_fixture_name, ndarray_object, request, checksum, decompress):
     """
     Given: Path to a FITS file
     When: Decoding the path the fits_array_decoder
     Then: The correct data are returned
     """
-    array = fits_array_decoder(path_to_primary_fits_file)
-    assert np.array_equal(ndarray_object, array)
+    file_path = request.getfixturevalue(path_fixture_name)
+    array = fits_array_decoder(
+        file_path, checksum=checksum, disable_image_compression=not decompress
+    )
+    if "compressed" in path_fixture_name and not decompress:
+        assert not np.array_equal(array, ndarray_object)
+    else:
+        assert np.array_equal(ndarray_object, array)
 
 
 def test_fits_array_decoder_autosqueeze(

dkist_processing_common/tests/test_flower_pot.py

@@ -4,11 +4,13 @@ from typing import Hashable
 import pytest
 
 from dkist_processing_common.models.flower_pot import FlowerPot
+from dkist_processing_common.models.flower_pot import ListStem
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.models.flower_pot import Stem
 
 
-@pytest.fixture()
+@pytest.fixture
 def simple_flower():
     class Flower(Stem):
         def setter(self, value: Any) -> Any:
@@ -22,7 +24,7 @@ def simple_flower():
     return Flower(stem_name="simple_flower")
 
 
-@pytest.fixture()
+@pytest.fixture
 def simple_flower_pot(simple_flower):
     flower_pot = FlowerPot()
     flower_pot.stems += [simple_flower]
@@ -30,16 +32,66 @@ def simple_flower_pot(simple_flower):
     return flower_pot
 
 
-@pytest.fixture()
+@pytest.fixture
 def simple_key_values():
     return {f"thing{i}": i for i in range(5)}
 
 
+@pytest.fixture
+def stem_bud():
+    class Bud(Stem):
+        def setter(self, value: int) -> int:
+            return value % 3
+
+        def getter(self, key: str) -> int:
+            return len(set(self.key_to_petal_dict.values()))
+
+    return Bud(stem_name="StemBud")
+
+
+@pytest.fixture
+def setstem_bud():
+    # Computes the same result as `stem_bud`
+    class SetBud(SetStem):
+        def setter(self, value: int) -> int:
+            return value % 3
+
+        def getter(self) -> int:
+            return len(self.value_set)
+
+    return SetBud(stem_name="SetStemBud")
+
+
+@pytest.fixture
+def liststem_bud():
+    # Highlights the difference between using a `set` and a `list` in these more efficient buds
+    class ListBud(ListStem):
+        def setter(self, value: int) -> int:
+            return value % 3
+
+        def getter(self) -> int:
+            return len(self.value_list)
+
+    return ListBud(stem_name="ListStemBud")
+
+
+@pytest.fixture
+def simple_bud_pot(stem_bud, setstem_bud, liststem_bud):
+    bud_pot = FlowerPot()
+    bud_pot.stems += [stem_bud, liststem_bud, setstem_bud]
+    return bud_pot
+
+
+@pytest.fixture
+def bud_key_values():
+    return {f"DOESN'T_MATTER_THAT'S_THE_POINT_{i}": i for i in [0, 1, 3, 4]}
+
+
 def test_simple_flower_pot(simple_flower_pot, simple_key_values):
     """
     Given: A FlowerPot with a simple Flower
-    When: Updating rock and flower with key: value pairs
-    Then: The rock and flower are correctly updated
+    When: Updating flower with key: value pairs
+    Then: The flower are correctly updated
     """
     assert len(simple_flower_pot) == 1
 
@@ -51,6 +103,8 @@ def test_simple_flower_pot(simple_flower_pot, simple_key_values):
 
     petals = sorted(list(flower.petals), key=lambda x: x.value)
     assert len(petals) == 2
+    assert flower.bud.value == petals[0].value
+    assert flower.bud.keys == petals[0].keys
     assert petals[0].value == 0
     assert petals[0].keys == ["thing0", "thing2", "thing4"]
     assert petals[1].value == 1
@@ -67,8 +121,12 @@ def test_cached_petal(simple_flower):
     value1 = 4
     simple_flower.update(key1, value1)
     assert len(simple_flower.petals) == 1
+
+    # Assert twice to hit the cache
     assert simple_flower.petals[0].value == value1 % 2  # % 2 because of simple_flower's `setter`
     assert simple_flower.petals[0].keys == [key1]
+    assert simple_flower.petals[0].value == value1 % 2
+    assert simple_flower.petals[0].keys == [key1]
 
     key2 = "thing2"
     value2 = 3
@@ -79,6 +137,10 @@ def test_cached_petal(simple_flower):
     assert sorted_petals[0].keys == [key1]
     assert sorted_petals[1].value == value2 % 2
     assert sorted_petals[1].keys == [key2]
+    assert sorted_petals[0].value == value1 % 2
+    assert sorted_petals[0].keys == [key1]
+    assert sorted_petals[1].value == value2 % 2
+    assert sorted_petals[1].keys == [key2]
 
 
 def test_spilled_dirt_flower(simple_flower):
@@ -103,3 +165,83 @@ def test_unhashable_dirt(simple_flower_pot):
     value = "never gonna get here"
     with pytest.raises(TypeError):
         simple_flower_pot.add_dirt(unhashable_key, value)
+
+
+def test_buds(simple_bud_pot, bud_key_values):
+    """
+    Given: A Flower pot with two Buds that compute the same thing; one a `Stem` and one a `SetStem`, and a `ListStem` bud
+    When: Updating the pot with key: value pairs
+    Then: The computed buds are correct and the `Stem` and `SetStem` buds match
+    """
+    assert len(simple_bud_pot) == 3
+
+    assert simple_bud_pot[0].stem_name == "StemBud"
+    assert simple_bud_pot[1].stem_name == "ListStemBud"
+    assert simple_bud_pot[2].stem_name == "SetStemBud"
+
+    for k, m in bud_key_values.items():
+        simple_bud_pot.add_dirt(k, m)
+
+    assert simple_bud_pot[0].bud.value == 2
+    assert simple_bud_pot[1].bud.value == 4
+    assert simple_bud_pot[2].bud.value == simple_bud_pot[0].bud.value
+
+    assert len(simple_bud_pot[0].petals) == 1
+    assert simple_bud_pot[0].petals[0].value == 2
+    assert simple_bud_pot[0].petals[0].keys == [
+        f"DOESN'T_MATTER_THAT'S_THE_POINT_{i}" for i in [0, 1, 3, 4]
+    ]
+
+    with pytest.raises(
+        AttributeError,
+        match="ListBud subclasses ListStem and therefore does not define the `petals` property",
+    ):
+        _ = simple_bud_pot[1].petals
+
+    with pytest.raises(
+        AttributeError,
+        match="SetBud subclasses SetStem and therefore does not define the `petals` property",
+    ):
+        _ = simple_bud_pot[2].petals
+
+
+def test_liststem_cached_bud(liststem_bud):
+    """
+    Given: A `ListStem` instance and some different input values
+    When: Computing the `bud` property after each value is ingested
+    Then: The `bud` value correctly updates based on the state of the `ListStem` object
+    """
+    key = "Who cares"
+    value1 = 3
+    liststem_bud.update(key, value1)
+
+    # Assert twice so we hit the cache
+    assert liststem_bud.bud.value == 1
+    assert liststem_bud.bud.value == 1
+
+    value2 = 1
+    liststem_bud.update(key, value2)
+
+    assert liststem_bud.bud.value == 2
+    assert liststem_bud.bud.value == 2
+
+
+def test_setstem_cached_bud(setstem_bud):
+    """
+    Given: A `SetStem` instance and some different input values
+    When: Computing the `bud` property after each value is ingested
+    Then: The `bud` value correctly updates based on the state of the `SetStem` object
+    """
+    key = "Who cares"
+    value1 = 3
+    setstem_bud.update(key, value1)
+
+    # Assert twice so we hit the cache
+    assert setstem_bud.bud.value == 1
+    assert setstem_bud.bud.value == 1
+
+    value2 = 1
+    setstem_bud.update(key, value2)
+
+    assert setstem_bud.bud.value == 2
+    assert setstem_bud.bud.value == 2

dkist_processing_common/tests/test_parse_l0_input_data.py

@@ -12,6 +12,8 @@ from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
 from dkist_processing_common.models.constants import BudName
 from dkist_processing_common.models.fits_access import FitsAccessBase
+from dkist_processing_common.models.flower_pot import ListStem
+from dkist_processing_common.models.flower_pot import SetStem
 from dkist_processing_common.models.flower_pot import SpilledDirt
 from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.flower_pot import Thorn
@@ -156,7 +158,27 @@ def empty_buds():
         def getter(self, key):
             pass  # We'll never get here because we spilled the dirt
 
-    return [EmptyBud()]
+    class EmptyListBud(ListStem):
+        def __init__(self):
+            super().__init__(stem_name="EMPTY_LIST_BUD")
+
+        def setter(self, value):
+            return SpilledDirt
+
+        def getter(self):
+            pass
+
+    class EmptySetBud(SetStem):
+        def __init__(self):
+            super().__init__(stem_name="EMPTY_SET_BUD")
+
+        def setter(self, value):
+            return SpilledDirt
+
+        def getter(self):
+            pass
+
+    return [EmptyBud(), EmptyListBud(), EmptySetBud()]
 
 
 @pytest.fixture()
@@ -266,13 +288,15 @@ def test_make_flowerpots(parse_inputs_task):
     tag_pot, constant_pot = parse_inputs_task.make_flower_pots()
 
     assert len(tag_pot.stems) == 2
-    assert len(constant_pot.stems) == 4
+    assert len(constant_pot.stems) == 6
     assert tag_pot.stems[0].stem_name == StemName.modstate
     assert tag_pot.stems[1].stem_name == "EMPTY_FLOWER"
     assert constant_pot.stems[0].stem_name == BudName.num_modstates
     assert constant_pot.stems[1].stem_name == "LOOKUP_BUD"
     assert constant_pot.stems[2].stem_name == "EMPTY_BUD"
-    assert constant_pot.stems[3].stem_name == "PICKY_BUD"
+    assert constant_pot.stems[3].stem_name == "EMPTY_LIST_BUD"
+    assert constant_pot.stems[4].stem_name == "EMPTY_SET_BUD"
+    assert constant_pot.stems[5].stem_name == "PICKY_BUD"
 
 
 def test_subclass_flowers(visp_parse_inputs_task, max_cs_step_time_sec):
@@ -336,7 +360,7 @@ def test_constants_correct(parse_inputs_task):
     parse_inputs_task.update_constants(constant_pot)
     assert dict(parse_inputs_task.constants._db_dict) == {
         BudName.num_modstates.value: parse_inputs_task._num_mod,
-        "LOOKUP_BUD": [[parse_inputs_task._num_mod, [0, 1]]],
+        "LOOKUP_BUD": {str(parse_inputs_task._num_mod): [0, 1]},
     }