dkist-processing-common 11.9.2__tar.gz → 11.9.3rc1__tar.gz

{dkist_processing_common-11.9.2 → dkist_processing_common-11.9.3rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 11.9.2
+Version: 11.9.3rc1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

dkist_processing_common-11.9.3rc1/changelog/281.feature.1.rst

@@ -0,0 +1,3 @@
+Add TimeLookupBud, which makes a dictionary that maps values of a time metadata key to values of another metadata key.
+Values of the time metadata key are rounded to avoid jitter.  Also add TaskTimeLookupBud, which makes the dictionary
+only for frames of a particular task type.

dkist_processing_common-11.9.3rc1/changelog/281.feature.rst

@@ -0,0 +1,2 @@
+Add a constant for dark number of frames per FPA.  The constant returns a dictionary of exposure times as keys
+and a list of number of frames per FPA as values.

{dkist_processing_common-11.9.2 → dkist_processing_common-11.9.3rc1}/dkist_processing_common/models/constants.py

@@ -53,6 +53,7 @@ class BudName(StrEnum):
     dark_date_end = "DARK_DATE_END"
     solar_gain_date_end = "SOLAR_GAIN_DATE_END"
     polcal_date_end = "POLCAL_DATE_END"
+    dark_num_raw_frames_per_fpa = "DARK_NUM_RAW_FRAMES_PER_FPA"
     solar_gain_num_raw_frames_per_fpa = "SOLAR_GAIN_NUM_RAW_FRAMES_PER_FPA"
     polcal_num_raw_frames_per_fpa = "POLCAL_NUM_RAW_FRAMES_PER_FPA"
     solar_gain_telescope_tracking_mode = "SOLAR_GAIN_TELESCOPE_TRACKING_MODE"
@@ -328,6 +329,13 @@ class ConstantsBase:
         """Return the date end constant for the polcal task."""
         return self._db_dict[BudName.polcal_date_end]
 
+    @property
+    def dark_num_raw_frames_per_fpa(self) -> dict[float, list]:
+        """Return the dictionary of exposure times to number of raw frames per fpa."""
+        raw_return = self._db_dict[BudName.dark_num_raw_frames_per_fpa]
+        # convert nested lists to dictionary
+        return {k: v for k, v in raw_return}
+
     @property
     def solar_gain_num_raw_frames_per_fpa(self) -> int:
         """Return the number of raw frames per fpa constant for the solar gain task."""

dkist_processing_common-11.9.3rc1/dkist_processing_common/parsers/lookup_bud.py

@@ -0,0 +1,129 @@
+"""Simple bud that is used to set a constant to a mapping dictionary."""
+
+from collections import defaultdict
+from enum import StrEnum
+from typing import Any
+from typing import Callable
+from typing import DefaultDict
+
+from dkist_processing_common.models.flower_pot import SpilledDirt
+from dkist_processing_common.models.flower_pot import Stem
+from dkist_processing_common.models.tags import EXP_TIME_ROUND_DIGITS
+from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
+from dkist_processing_common.parsers.task import passthrough_header_ip_task
+
+
+class TimeLookupBud(Stem):
+    """
+    Bud that reads two header keys from all files and creates a dictionary mapping a time KEY value to sets of a VALUE value.
+
+    Parameters
+    ----------
+    constant_name
+        The name for the constant to be defined
+
+    key_metadata_key
+        The time metadata key for the resulting dictionary key
+
+    value_metadata_key
+        The metadata key for the resulting dictionary value
+    """
+
+    def __init__(
+        self,
+        constant_name: str,
+        key_metadata_key: str | StrEnum,
+        value_metadata_key: str | StrEnum,
+    ):
+        super().__init__(stem_name=constant_name)
+
+        if isinstance(key_metadata_key, StrEnum):
+            key_metadata_key = key_metadata_key.name
+        self.key_metadata_key = key_metadata_key
+        if isinstance(value_metadata_key, StrEnum):
+            value_metadata_key = value_metadata_key.name
+        self.value_metadata_key = value_metadata_key
+
+        self.mapping: DefaultDict[float, set[Any]] = defaultdict(set)
+
+    def setter(self, fits_obj: L0FitsAccess):
+        """
+        Update the mapping dictionary.
+
+        Parameters
+        ----------
+        fits_obj
+            The input fits object
+        Returns
+        -------
+        Updates the dictionary and returns None
+        """
+        key = getattr(fits_obj, self.key_metadata_key)
+        rounded_key = round(key, EXP_TIME_ROUND_DIGITS)
+        value = getattr(fits_obj, self.value_metadata_key)
+        self.mapping[rounded_key].add(value)
+        return None
+
+    def getter(self, key):
+        """
+        Get the dictionary mapping created by the setter converted into hashable nested tuples.
+
+        Parameters
+        ----------
+        key
+            The input key
+        Returns
+        -------
+        The mapping dictionary converted into hashable nested tuples
+        """
+        hashable_mapping = tuple((k, tuple(v)) for k, v in self.mapping.items())
+        return hashable_mapping
+
+
+class TaskTimeLookupBud(TimeLookupBud):
+    """
+    Subclass of `TimeLookupBud` that only considers objects that have specific task types.
+
+    Parameters
+    ----------
+    constant_name
+        The name for the constant to be defined
+
+    key_metadata_key
+        The time metadata key for the resulting dictionary key
+
+    value_metadata_key
+        The metadata key for the resulting dictionary value
+
+    ip_task_types
+        Only consider objects whose parsed header IP task type matches a string in this list
+
+    task_type_parsing_function
+        The function used to convert a header into an IP task type
+    """
+
+    def __init__(
+        self,
+        constant_name: str,
+        key_metadata_key: str | StrEnum,
+        value_metadata_key: str | StrEnum,
+        ip_task_types: str | list[str],
+        task_type_parsing_function: Callable = passthrough_header_ip_task,
+    ):
+        super().__init__(
+            constant_name=constant_name,
+            key_metadata_key=key_metadata_key,
+            value_metadata_key=value_metadata_key,
+        )
+        if isinstance(ip_task_types, str):
+            ip_task_types = [ip_task_types]
+        self.ip_task_types = [task.casefold() for task in ip_task_types]
+        self.parsing_function = task_type_parsing_function
+
+    def setter(self, fits_obj: L0FitsAccess):
+        """Ingest an object only if its parsed IP task type matches what's desired."""
+        task = self.parsing_function(fits_obj)
+        if task.casefold() in self.ip_task_types:
+            return super().setter(fits_obj)
+
+        return SpilledDirt

{dkist_processing_common-11.9.2 → dkist_processing_common-11.9.3rc1}/dkist_processing_common/tasks/parse_l0_input_data.py

@@ -37,6 +37,7 @@ from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_common.parsers.average_bud import TaskAverageBud
 from dkist_processing_common.parsers.experiment_id_bud import ContributingExperimentIdsBud
 from dkist_processing_common.parsers.experiment_id_bud import ExperimentIdBud
+from dkist_processing_common.parsers.lookup_bud import TaskTimeLookupBud
 from dkist_processing_common.parsers.observing_program_id_bud import (
     TaskContributingObservingProgramExecutionIdsBud,
 )
@@ -103,6 +104,12 @@ def dataset_extra_bud_factory() -> list[S]:
             constant_name=BudName.polcal_observing_program_execution_ids,
             ip_task_types=TaskName.polcal,
         ),
+        TaskTimeLookupBud(
+            constant_name=BudName.dark_num_raw_frames_per_fpa,
+            key_metadata_key=MetadataKey.sensor_readout_exposure_time_ms,
+            value_metadata_key=MetadataKey.num_raw_frames_per_fpa,
+            ip_task_types=TaskName.dark,
+        ),
         TaskUniqueBud(
             constant_name=BudName.solar_gain_num_raw_frames_per_fpa,
             metadata_key=MetadataKey.num_raw_frames_per_fpa,

{dkist_processing_common-11.9.2 → dkist_processing_common-11.9.3rc1}/dkist_processing_common/tests/test_parse_l0_input_data.py

@@ -17,6 +17,7 @@ from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.flower_pot import Thorn
 from dkist_processing_common.models.tags import StemName
 from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.parsers.lookup_bud import TimeLookupBud
 from dkist_processing_common.parsers.single_value_single_key_flower import (
     SingleValueSingleKeyFlower,
 )
@@ -117,6 +118,17 @@ def visp_buds():
     return [UniqueBud(constant_name=BudName.num_modstates, metadata_key=ViSPMetadataKey.num_mod)]
 
 
+@pytest.fixture(scope="function")
+def visp_lookup_buds():
+    return [
+        TimeLookupBud(
+            constant_name="LOOKUP_BUD",
+            key_metadata_key=ViSPMetadataKey.num_mod,
+            value_metadata_key=ViSPMetadataKey.modstate,
+        )
+    ]
+
+
 @pytest.fixture(scope="function")
 def empty_flowers():
     class EmptyFlower(Stem):
@@ -161,8 +173,17 @@ def picky_buds():
 
 @pytest.fixture(scope="function")
 def parse_inputs_task(
-    tmp_path, visp_flowers, visp_buds, empty_flowers, empty_buds, picky_buds, recipe_run_id
+    tmp_path,
+    visp_flowers,
+    visp_buds,
+    visp_lookup_buds,
+    empty_flowers,
+    empty_buds,
+    picky_buds,
+    recipe_run_id,
 ):
+    """Override parse task class and make data for testing."""
+
     class TaskClass(ParseL0InputDataBase):
         @property
         def tag_flowers(self):
@@ -170,7 +191,7 @@ def parse_inputs_task(
 
         @property
         def constant_buds(self):
-            return visp_buds + empty_buds + picky_buds
+            return visp_buds + visp_lookup_buds + empty_buds + picky_buds
 
         @property
         def fits_parsing_class(self):
@@ -210,6 +231,8 @@ def parse_inputs_task(
 
 @pytest.fixture()
 def visp_parse_inputs_task(tmp_path, visp_flowers, visp_buds, recipe_run_id):
+    """Extend parse task class, but don't make data for testing."""
+
     class TaskClass(ParseL0InputDataBase):
         @property
         def tag_flowers(self):
@@ -243,12 +266,13 @@ def test_make_flowerpots(parse_inputs_task):
     tag_pot, constant_pot = parse_inputs_task.make_flower_pots()
 
     assert len(tag_pot.stems) == 2
-    assert len(constant_pot.stems) == 3
+    assert len(constant_pot.stems) == 4
     assert tag_pot.stems[0].stem_name == StemName.modstate
     assert tag_pot.stems[1].stem_name == "EMPTY_FLOWER"
     assert constant_pot.stems[0].stem_name == BudName.num_modstates
-    assert constant_pot.stems[1].stem_name == "EMPTY_BUD"
-    assert constant_pot.stems[2].stem_name == "PICKY_BUD"
+    assert constant_pot.stems[1].stem_name == "LOOKUP_BUD"
+    assert constant_pot.stems[2].stem_name == "EMPTY_BUD"
+    assert constant_pot.stems[3].stem_name == "PICKY_BUD"
 
 
 def test_subclass_flowers(visp_parse_inputs_task, max_cs_step_time_sec):
@@ -260,7 +284,7 @@ def test_subclass_flowers(visp_parse_inputs_task, max_cs_step_time_sec):
     tag_pot, constant_pot = visp_parse_inputs_task.make_flower_pots()
 
     assert len(tag_pot.stems) == 1
-    assert len(constant_pot.stems) == 60
+    assert len(constant_pot.stems) == 61
     all_flower_names = [StemName.modstate]
     assert sorted([f.stem_name for f in tag_pot.stems]) == sorted(all_flower_names)
     all_bud_names = [b.stem_name for b in default_constant_bud_factory()] + [BudName.num_modstates]
@@ -296,27 +320,23 @@ def test_dataset_extra_bud_factory(visp_parse_inputs_task, max_cs_step_time_sec)
     ]
     for base in bud_name_base:
         assert "SOLAR_GAIN_" + base in stem_names
-        if base not in [
-            "NUM_RAW_FRAMES_PER_FPA",
-            "TELESCOPE_TRACKING_MODE",
-            "COUDE_TABLE_TRACKING_MODE",
-            "TELESCOPE_SCANNING_MODE",
-        ]:
-            assert "DARK_" + base in stem_names
-        if "GOS" not in base:
-            assert "POLCAL_" + base in stem_names
+        # telescope mode keys are not constant for dark frames
+        assert ("DARK_" + base in stem_names) ^ ("MODE" in base)
+        # gos keys are not constant for polcal frames
+        assert ("POLCAL_" + base in stem_names) ^ ("GOS" in base)
 
 
 def test_constants_correct(parse_inputs_task):
     """
     Given: ParseInputData task with a populated constant FlowerPot
     When: Updating pipeline constants
-    Then: A pipeline constant is correctly populated
+    Then: A pipeline constant is correctly populated and the values return correctly
     """
     _, constant_pot = parse_inputs_task.make_flower_pots()
     parse_inputs_task.update_constants(constant_pot)
     assert dict(parse_inputs_task.constants._db_dict) == {
         BudName.num_modstates.value: parse_inputs_task._num_mod,
+        "LOOKUP_BUD": [[parse_inputs_task._num_mod, [0, 1]]],
     }
 
 

{dkist_processing_common-11.9.2 → dkist_processing_common-11.9.3rc1}/dkist_processing_common/tests/test_stems.py

@@ -1,3 +1,4 @@
+import collections
 from enum import StrEnum
 from itertools import chain
 
@@ -17,6 +18,8 @@ from dkist_processing_common.parsers.dsps_repeat import TotalDspsRepeatsBud
 from dkist_processing_common.parsers.experiment_id_bud import ContributingExperimentIdsBud
 from dkist_processing_common.parsers.experiment_id_bud import ExperimentIdBud
 from dkist_processing_common.parsers.id_bud import TaskContributingIdsBud
+from dkist_processing_common.parsers.lookup_bud import TaskTimeLookupBud
+from dkist_processing_common.parsers.lookup_bud import TimeLookupBud
 from dkist_processing_common.parsers.near_bud import NearFloatBud
 from dkist_processing_common.parsers.near_bud import TaskNearFloatBud
 from dkist_processing_common.parsers.observing_program_id_bud import (
@@ -203,7 +206,7 @@ def basic_header_objs():
                 "ID___012": "experiment_id_1",
                 "XPOSURE": 100.0,
                 "TEXPOSUR": 11.0,
-                "NSUMEXP": 4,
+                "NSUMEXP": 5,
                 "DSPSNUM": 2,
                 "DSPSREPS": 2,
                 "DATE-OBS": "2022-06-17T22:00:03.000",
@@ -1087,4 +1090,49 @@ def test_task_average_bud(basic_header_objs):
     assert round(petal[0].value, 3) == 1.227
 
 
+def test_time_lookup_bud(basic_header_objs):
+    """
+    Given: A set of headers with two differently valued header keys
+    When: Ingesting headers with a TimeLookupBud and asking for the value
+    Then: The bud's value is a dictionary of one key to sets of the other key as nested tuples
+    """
+    bud = TimeLookupBud(
+        constant_name="lookup",
+        key_metadata_key=FitsReaderMetadataKey.fpa_exposure_time_ms,
+        value_metadata_key=FitsReaderMetadataKey.num_raw_frames_per_fpa,
+    )
+    assert bud.stem_name == "lookup"
+    for fo in basic_header_objs:
+        key = fo.name
+        bud.update(key, fo)
+
+    assert type(bud.mapping) == collections.defaultdict
+    assert bud.mapping == {0.0013: {3}, 12.345: {1}, 100.0: {4, 5}}
+    assert len(bud.petals) == 1
+    expected_value = ((0.0013, (3,)), (12.345, (1,)), (100.0, (4, 5)))
+    assert bud.petals[0].value == expected_value
+
+
+def test_task_time_lookup_bud(basic_header_objs):
+    """
+    Given: A set of headers with two differently valued header keys
+    When: Ingesting headers with a TaskTimeLookupBud and asking for the value
+    Then: The bud's value is a dictionary of one key to sets of the other key as nested tuples
+    """
+    bud = TaskTimeLookupBud(
+        constant_name="task_lookup",
+        key_metadata_key=FitsReaderMetadataKey.fpa_exposure_time_ms,
+        value_metadata_key=FitsReaderMetadataKey.num_raw_frames_per_fpa,
+        ip_task_types="dark",
+    )
+    assert bud.stem_name == "task_lookup"
+    for fo in basic_header_objs:
+        key = fo.name
+        bud.update(key, fo)
+
+    assert len(bud.petals) == 1
+    expected_value = ((12.345, (1,)),)
+    assert bud.petals[0].value == expected_value
+
+
 # TODO: test new stem types that have been added to parse_l0_input_data

{dkist_processing_common-11.9.2 → dkist_processing_common-11.9.3rc1}/dkist_processing_common/tests/test_workflow_task_base.py

@@ -175,6 +175,17 @@ def test_constants(workflow_data_task):
     assert task.constants.instrument_twice == "foofoo"
 
 
+def test_constant_dark_num_raw_frames_per_fpa(workflow_data_task):
+    """
+    Given: a WorkflowDataTask's constants object
+    When: accessing a constant that does unusual processing in the property
+    Then: the value is correctly processed
+    """
+    task = workflow_data_task[0]
+    task.constants._update({BudName.dark_num_raw_frames_per_fpa: ((1.1, (2, 3)), (4.4, (5,)))})
+    assert task.constants.dark_num_raw_frames_per_fpa == {1.1: [2, 3], 4.4: [5]}
+
+
 def test_dataset_id(workflow_data_task):
     """
     Given: a ParsedL0InputTaskBase task

{dkist_processing_common-11.9.2 → dkist_processing_common-11.9.3rc1}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 11.9.2
+Version: 11.9.3rc1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-11.9.2 → dkist_processing_common-11.9.3rc1}/dkist_processing_common.egg-info/SOURCES.txt

@@ -7,6 +7,8 @@ README.rst
 bitbucket-pipelines.yml
 pyproject.toml
 changelog/.gitempty
+changelog/281.feature.1.rst
+changelog/281.feature.rst
 dkist_processing_common/__init__.py
 dkist_processing_common/config.py
 dkist_processing_common/manual.py
@@ -58,6 +60,7 @@ dkist_processing_common/parsers/experiment_id_bud.py
 dkist_processing_common/parsers/id_bud.py
 dkist_processing_common/parsers/l0_fits_access.py
 dkist_processing_common/parsers/l1_fits_access.py
+dkist_processing_common/parsers/lookup_bud.py
 dkist_processing_common/parsers/near_bud.py
 dkist_processing_common/parsers/observing_program_id_bud.py
 dkist_processing_common/parsers/proposal_id_bud.py