dkist-processing-cryonirsp 0.0.47__tar.gz → 0.0.49__tar.gz

{dkist_processing_cryonirsp-0.0.47 → dkist_processing_cryonirsp-0.0.49}/CHANGELOG.rst

@@ -1,3 +1,22 @@
+v0.0.49 (2024-03-15)
+====================
+
+Bugfixes
+--------
+
+- Correctly identify *partially* incomplete scan steps. Previously any scan step that had at least a single file was
+  considered to exist in its entirety, even if some of its files were missing. (`#108 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/108>`__)
+
+
+v0.0.48 (2024-03-15)
+====================
+
+Bugfixes
+--------
+
+- Fixes a bug in `ParseL0CryonirspLinearizedData` which conflated dark frames with the same exposure time but different OD filters. We need to be able to identify dark frame based on their exposure conditions (exposure time, Optical Density Filter) and use the exposure conditions to correlate the sets of dark frames with the frames sets they will be used to correct (lamp gain, solar gain, observe). (`#106 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/106>`__)
+
+
 v0.0.47 (2024-03-15)
 ====================
 

{dkist_processing_cryonirsp-0.0.47 → dkist_processing_cryonirsp-0.0.49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist_processing_cryonirsp
-Version: 0.0.47
+Version: 0.0.49
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA

{dkist_processing_cryonirsp-0.0.47 → dkist_processing_cryonirsp-0.0.49}/bitbucket-pipelines.yml

@@ -50,7 +50,7 @@ definitions:
         script:
           - pip install -U pip
           - pip install .[test]
-          - pytest -v -n auto --cov -m "not development" dkist_processing_cryonirsp
+          - pytest -v -n auto --dist worksteal --cov -m "not development" dkist_processing_cryonirsp
         services:
           - redis
     - step: &push_workflow

{dkist_processing_cryonirsp-0.0.47 → dkist_processing_cryonirsp-0.0.49}/dkist_processing_cryonirsp/models/constants.py

@@ -5,6 +5,8 @@ from enum import unique
 from dkist_processing_common.models.constants import BudName
 from dkist_processing_common.models.constants import ConstantsBase
 
+from dkist_processing_cryonirsp.models.exposure_conditions import ExposureConditions
+
 
 @unique
 class CryonirspBudName(Enum):
@@ -19,11 +21,16 @@ class CryonirspBudName(Enum):
     camera_readout_mode = "CAM_READOUT_MODE"
     num_meas = "NUM_MEAS"
     time_obs_list = "TIME_OBS_LIST"
-    lamp_gain_exposure_times = "LAMP_GAIN_EXPOSURE_TIMES"
-    solar_gain_exposure_times = "SOLAR_GAIN_EXPOSURE_TIMES"
-    polcal_exposure_times = "POLCAL_EXPOSURE_TIMES"
-    observe_exposure_times = "OBSERVE_EXPOSURE_TIMES"
-    non_dark_and_non_polcal_task_exposure_times = "NON_DARK_AND_NON_POLCAL_TASK_EXPOSURE_TIMES"
+    exposure_conditions_list = "EXPOSURE_CONDITIONS_LIST"
+    dark_frame_exposure_conditions_list = "DARK_FRAME_EXPOSURE_CONDITIONS_LIST"
+    lamp_gain_exposure_conditions_list = "LAMP_GAIN_EXPOSURE_CONDITIONS_LIST"
+    solar_gain_exposure_conditions_list = "SOLAR_GAIN_EXPOSURE_CONDITIONS_LIST"
+    polcal_exposure_conditions_list = "POLCAL_EXPOSURE_CONDITIONS_LIST"
+    observe_exposure_conditions_list = "OBSERVE_EXPOSURE_CONDITIONS_LIST"
+    non_dark_and_non_polcal_task_exposure_conditions_list = (
+        "NON_DARK_AND_NON_POLCAL_TASK_EXPOSURE_CONDITIONS_LIST"
+    )
+    picky_dark_exposure_conditions_list = "PICKY_DARK_EXPOSURE_CONDITIONS_LIST"
     modulator_spin_mode = "MODULATOR_SPIN_MODE"
     axis_1_type = "AXIS_1_TYPE"
     axis_2_type = "AXIS_2_TYPE"
@@ -32,7 +39,6 @@ class CryonirspBudName(Enum):
     roi_1_origin_y = "ROI_1_ORIGIN_Y"
     roi_1_size_x = "ROI_1_SIZE_X"
     roi_1_size_y = "ROI_1_SIZE_Y"
-    picky_dark_exposure_times = "PICKY_DARK_EXPOSURE_TIMES"
     optical_density_filter_picky_bud = "OPTICAL_DENSITY_FILTER_PICKY_BUD"
 
 
@@ -83,32 +89,70 @@ class CryonirspConstants(ConstantsBase):
         return self._db_dict[CryonirspBudName.time_obs_list.value]
 
     @property
-    def lamp_gain_exposure_times(self) -> [float]:
-        """Construct a list of lamp gain exposure times for the dataset."""
-        return self._db_dict[CryonirspBudName.lamp_gain_exposure_times.value]
+    def exposure_conditions_list(self) -> [ExposureConditions]:
+        """Construct a list of ExposureConditions tuples for the dataset."""
+        raw_conditions: list[list[int, str]] = self._db_dict[
+            CryonirspBudName.exposure_conditions.value
+        ]
+        conditions = [ExposureConditions(*item) for item in raw_conditions]
+        return conditions
+
+    @property
+    def dark_exposure_conditions_list(self) -> [ExposureConditions]:
+        """Construct a list of dark frame ExposureConditions tuples for the dataset."""
+        raw_conditions: list[list[int, str]] = self._db_dict[
+            CryonirspBudName.dark_frame_exposure_conditions_list.value
+        ]
+        conditions = [ExposureConditions(*item) for item in raw_conditions]
+        return conditions
+
+    @property
+    def lamp_gain_exposure_conditions_list(self) -> [ExposureConditions]:
+        """Construct a list of lamp gain ExposureConditions tuples for the dataset."""
+        raw_conditions: list[list[int, str]] = self._db_dict[
+            CryonirspBudName.lamp_gain_exposure_conditions_list.value
+        ]
+        conditions = [ExposureConditions(*item) for item in raw_conditions]
+        return conditions
 
     @property
-    def solar_gain_exposure_times(self) -> [float]:
-        """Construct a list of solar gain exposure times for the dataset."""
-        return self._db_dict[CryonirspBudName.solar_gain_exposure_times.value]
+    def solar_gain_exposure_conditions_list(self) -> [ExposureConditions]:
+        """Construct a list of solar gain ExposureConditions tuples for the dataset."""
+        raw_conditions: list[list[int, str]] = self._db_dict[
+            CryonirspBudName.solar_gain_exposure_conditions_list.value
+        ]
+        conditions = [ExposureConditions(*item) for item in raw_conditions]
+        return conditions
 
     @property
-    def polcal_exposure_times(self) -> [float]:
-        """Construct a list of polcal exposure times for the dataset."""
+    def observe_exposure_conditions_list(self) -> [ExposureConditions]:
+        """Construct a list of observe ExposureConditions tuples for the dataset."""
+        raw_conditions: list[list[int, str]] = self._db_dict[
+            CryonirspBudName.observe_exposure_conditions_list.value
+        ]
+        conditions = [ExposureConditions(*item) for item in raw_conditions]
+        return conditions
+
+    @property
+    def polcal_exposure_conditions_list(self) -> [ExposureConditions]:
+        """Construct a list of polcal ExposureConditions tuples for the dataset."""
         if self.correct_for_polarization:
-            return self._db_dict[CryonirspBudName.polcal_exposure_times.value]
+            raw_conditions: list[list[int, str]] = self._db_dict[
+                CryonirspBudName.polcal_exposure_conditions_list.value
+            ]
+            conditions = [ExposureConditions(*item) for item in raw_conditions]
+            return conditions
         else:
             return []
 
     @property
-    def observe_exposure_times(self) -> [float]:
-        """Construct a list of observe exposure times."""
-        return self._db_dict[CryonirspBudName.observe_exposure_times.value]
-
-    @property
-    def non_dark_and_non_polcal_task_exposure_times(self) -> [float]:
+    def non_dark_and_non_polcal_task_exposure_conditions_list(self) -> [ExposureConditions]:
         """Return a list of all exposure times required for all tasks other than dark and polcal."""
-        return self._db_dict[CryonirspBudName.non_dark_and_non_polcal_task_exposure_times.value]
+        raw_conditions: list[list[int, str]] = self._db_dict[
+            CryonirspBudName.non_dark_and_non_polcal_task_exposure_conditions_list.value
+        ]
+        conditions = [ExposureConditions(*item) for item in raw_conditions]
+        return conditions
 
     @property
     def num_modstates(self) -> int:

dkist_processing_cryonirsp-0.0.49/dkist_processing_cryonirsp/models/exposure_conditions.py

@@ -0,0 +1,26 @@
+"""Support classes for exposure conditions."""
+from collections import namedtuple
+from enum import StrEnum
+
+# Number of digits used to round the exposure when creating the ExposureConditions tuple in fits_access
+CRYO_EXP_TIME_ROUND_DIGITS: int = 3
+
+"""Base class to hold a tuple of exposure time and filter name."""
+ExposureConditionsBase = namedtuple("ExposureConditions", ["exposure_time", "filter_name"])
+
+
+class ExposureConditions(ExposureConditionsBase):
+    """Define str to make tags look reasonable."""
+
+    def __str__(self):
+        return f"{self.exposure_time}_{self.filter_name}"
+
+
+class AllowableOpticalDensityFilterNames(StrEnum):
+    """Enum to implement list of allowable Optical Density Filter names."""
+
+    G278 = "G278"
+    G358 = "G358"
+    G408 = "G408"
+    OPEN = "OPEN"
+    NONE = "NONE"

{dkist_processing_cryonirsp-0.0.47 → dkist_processing_cryonirsp-0.0.49}/dkist_processing_cryonirsp/models/parameters.py

@@ -6,9 +6,7 @@ import numpy as np
 from dkist_processing_common.models.parameters import ParameterBase
 from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetParameterValue
 
-from dkist_processing_cryonirsp.models.optical_density_filter_names import (
-    AllowableOpticalDensityFilterNames,
-)
+from dkist_processing_cryonirsp.models.exposure_conditions import AllowableOpticalDensityFilterNames
 
 
 class CryonirspParameters(ParameterBase):

{dkist_processing_cryonirsp-0.0.47 → dkist_processing_cryonirsp-0.0.49}/dkist_processing_cryonirsp/models/tags.py

@@ -5,6 +5,7 @@ from dkist_processing_common.models.tags import StemName
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.models.task_name import TaskName
 
+from dkist_processing_cryonirsp.models.exposure_conditions import ExposureConditions
 from dkist_processing_cryonirsp.models.task_name import CryonirspTaskName
 
 
@@ -19,6 +20,7 @@ class CryonirspStemName(str, Enum):
     meas_num = "MEAS_NUM"
     modstate = "MODSTATE"
     map_scan = "MAP_SCAN"
+    exposure_conditions = "EXPOSURE_CONDITIONS"
 
 
 class CryonirspTag(Tag):
@@ -147,6 +149,22 @@ class CryonirspTag(Tag):
         """
         return cls.format_tag(CryonirspStemName.meas_num, meas_num)
 
+    @classmethod
+    def exposure_conditions(cls, exposure_conditions: ExposureConditions) -> str:
+        """
+        Tags by the measurement number.
+
+        Parameters
+        ----------
+        exposure_conditions
+            A tuple of (exposure time, filter name)
+
+        Returns
+        -------
+        The formatted tag string
+        """
+        return cls.format_tag(CryonirspStemName.exposure_conditions, exposure_conditions)
+
     @classmethod
     def task_beam_boundaries(cls) -> str:
         """Tags intermediate beam boundary calibration objects."""

{dkist_processing_cryonirsp-0.0.47 → dkist_processing_cryonirsp-0.0.49}/dkist_processing_cryonirsp/parsers/cryonirsp_l0_fits_access.py

@@ -2,6 +2,9 @@
 from astropy.io import fits
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 
+from dkist_processing_cryonirsp.models.exposure_conditions import CRYO_EXP_TIME_ROUND_DIGITS
+from dkist_processing_cryonirsp.models.exposure_conditions import ExposureConditions
+
 
 class CryonirspRampFitsAccess(L0FitsAccess):
     """
@@ -84,6 +87,11 @@ class CryonirspL0FitsAccess(L0FitsAccess):
         self.axis_2_type: str = self.header["CTYPE2"]
         self.axis_3_type: str = self.header["CTYPE3"]
         self.obs_ip_start_time = self.header["DKIST011"]
+        # The ExposureConditions are a combination of the exposure time and the OD filter name:
+        self.exposure_conditions = ExposureConditions(
+            round(self.fpa_exposure_time_ms, CRYO_EXP_TIME_ROUND_DIGITS),
+            self.header["CNFILTNP"].upper(),
+        )
 
     @property
     def cn1_scan_step(self):

dkist_processing_cryonirsp-0.0.49/dkist_processing_cryonirsp/parsers/exposure_conditions.py

@@ -0,0 +1,163 @@
+"""Buds to parse the combination of exposure time and filter name."""
+from collections import namedtuple
+from typing import Hashable
+from typing import Type
+
+from dkist_processing_common.models.flower_pot import SpilledDirt
+from dkist_processing_common.models.flower_pot import Stem
+from dkist_processing_common.models.flower_pot import Thorn
+from dkist_processing_common.models.task_name import TaskName
+from dkist_processing_common.parsers.task import parse_header_ip_task_with_gains
+
+from dkist_processing_cryonirsp.models.constants import CryonirspBudName
+from dkist_processing_cryonirsp.models.exposure_conditions import ExposureConditions
+from dkist_processing_cryonirsp.parsers.cryonirsp_l0_fits_access import CryonirspL0FitsAccess
+
+_DARK_AND_POLCAL_TASK_TYPES = (TaskName.dark.value, TaskName.polcal.value)
+
+
+class CryonirspTaskExposureConditionsBud(Stem):
+    """
+    Bud to allow custom parsing of exposure conditions based on ip task type.
+
+    Parameters
+    ----------
+    stem_name : str
+        The name of the stem of the tag
+    ip_task_type : str
+        Instrument program task type
+    """
+
+    def __init__(self, stem_name: str, ip_task_type: str):
+        super().__init__(stem_name=stem_name)
+        self.metadata_key = "exposure_conditions"
+        self.ip_task_type = ip_task_type
+
+    def setter(self, fits_obj: CryonirspL0FitsAccess):
+        """
+        Set the value of the bud.
+
+        Parameters
+        ----------
+        fits_obj:
+            A single FitsAccess object
+        """
+        ip_task_type = parse_header_ip_task_with_gains(fits_obj)
+        if ip_task_type.lower() == self.ip_task_type.lower():
+            return getattr(fits_obj, self.metadata_key)
+        return SpilledDirt
+
+    def getter(self, key: Hashable) -> tuple[ExposureConditions, ...]:
+        """Return a list of the ExposureConditions for this ip task type."""
+        exposure_conditions_tuple = tuple(sorted(set(self.key_to_petal_dict.values())))
+        return exposure_conditions_tuple
+
+
+class CryonirspNonDarkAndNonPolcalTaskExposureConditionsBud(Stem):
+    """For ip task types that are neither DARK nor POLCAL, produce a list of exposure conditions tuples."""
+
+    def __init__(self):
+        super().__init__(
+            stem_name=CryonirspBudName.non_dark_and_non_polcal_task_exposure_conditions_list.value
+        )
+        self.metadata_key = "exposure_conditions"
+
+    def setter(self, fits_obj: CryonirspL0FitsAccess) -> ExposureConditions | Type[SpilledDirt]:
+        """
+        Set the task exposure conditions tuple for this fits object.
+
+        Parameters
+        ----------
+        fits_obj
+            The input fits object
+        Returns
+        -------
+        The exposure time associated with this fits object
+        """
+        if fits_obj.ip_task_type.upper() not in _DARK_AND_POLCAL_TASK_TYPES:
+            return getattr(fits_obj, self.metadata_key)
+        return SpilledDirt
+
+    def getter(self, key: Hashable) -> tuple[ExposureConditions, ...]:
+        """
+        Get the list of exposure conditions tuples.
+
+        Parameters
+        ----------
+        key
+            The input key
+
+        Returns
+        -------
+        A tuple of (exposure time, filter name) tuples
+        """
+        exposure_conditions_tuple = tuple(sorted(set(self.key_to_petal_dict.values())))
+        return exposure_conditions_tuple
+
+
+class CryonirspPickyDarkExposureConditionsBud(Stem):
+    """Parse exposure conditions tuples to ensure existence of dark frames with the required exposure conditions."""
+
+    DarkTaskTestAndExposureConditions = namedtuple(
+        "DarkTaskTestAndExposureConditions", ["is_dark", "exposure_conditions"]
+    )
+
+    def __init__(self):
+        super().__init__(stem_name=CryonirspBudName.picky_dark_exposure_conditions_list.value)
+        self.metadata_key = "exposure_conditions"
+
+    def setter(self, fits_obj: CryonirspL0FitsAccess) -> tuple | Type[SpilledDirt]:
+        """
+        Set the task exposure conditions tuple and whether it is a DARK task for this fits object.
+
+        Parameters
+        ----------
+        fits_obj
+            The input fits object
+        Returns
+        -------
+        A tuple of a boolean indicating if the task is a dark task, and the exposure conditions for this fits object
+        """
+        if fits_obj.ip_task_type.upper() == TaskName.dark.value:
+            return self.DarkTaskTestAndExposureConditions(
+                is_dark=True, exposure_conditions=getattr(fits_obj, self.metadata_key)
+            )
+        if fits_obj.ip_task_type.upper() not in _DARK_AND_POLCAL_TASK_TYPES:
+            return self.DarkTaskTestAndExposureConditions(
+                is_dark=False, exposure_conditions=getattr(fits_obj, self.metadata_key)
+            )
+            # Polcal falls through
+        return SpilledDirt
+
+    def getter(self, key: Hashable) -> Type[Thorn]:
+        """
+        Parse all exposure conditions and raise an error if any non-dark exposure condition is missing from the set of dark exposure conditions.
+
+        Parameters
+        ----------
+        key
+            The input key
+
+        Returns
+        -------
+        Thorn
+        """
+        dark_task_test_and_exposure_conditions_set = set(self.key_to_petal_dict.values())
+        dark_exposure_conditions_set = {
+            item.exposure_conditions
+            for item in dark_task_test_and_exposure_conditions_set
+            if item.is_dark
+        }
+        other_exposure_conditions_set = {
+            item.exposure_conditions
+            for item in dark_task_test_and_exposure_conditions_set
+            if not item.is_dark
+        }
+        other_exposure_conditions_missing_from_dark_exposure_conditions = (
+            other_exposure_conditions_set - dark_exposure_conditions_set
+        )
+        if other_exposure_conditions_missing_from_dark_exposure_conditions:
+            raise ValueError(
+                f"Exposure conditions required in the set of dark frames not found. Missing conditions = {other_exposure_conditions_missing_from_dark_exposure_conditions}"
+            )
+        return Thorn

dkist_processing_cryonirsp-0.0.49/dkist_processing_cryonirsp/parsers/map_repeats.py

@@ -0,0 +1,40 @@
+"""Stems for organizing files into separate dsps repeats."""
+from dkist_processing_cryonirsp.models.constants import CryonirspBudName
+from dkist_processing_cryonirsp.models.tags import CryonirspStemName
+from dkist_processing_cryonirsp.parsers.scan_step import MapScanStepStemBase
+
+
+class MapScanFlower(MapScanStepStemBase):
+    """Flower for computing and assigning map scan numbers."""
+
+    def __init__(self):
+        super().__init__(stem_name=CryonirspStemName.map_scan.value)
+
+    def getter(self, key: str) -> int:
+        """Compute the map scan number for a single frame.
+
+        The frame implies a SingleScanStep. That object is then compared to the sorted list of objects for a single
+        (raster_step, meas_num, modstate, sub_repeat) tuple. The location within that sorted list is the map scan number.
+        """
+        return self.get_map_scan_for_key(key)
+
+
+class NumMapScansBud(MapScanStepStemBase):
+    """
+    Bud for determining the total number of dsps repeats.
+
+    Also checks that all scan steps have the same number of dsps repeats.
+    """
+
+    def __init__(self):
+        super().__init__(stem_name=CryonirspBudName.num_map_scans.value)
+
+    def getter(self, key: str) -> int:
+        """
+        Compute the total number of dsps repeats.
+
+        The number of map_scans for every scan step are calculated and if a map_scan is incomplete,
+        it will not be included.
+        Assumes the incomplete map_scan is always the last one due to summit abort or cancellation.
+        """
+        return self.number_of_complete_map_scans

{dkist_processing_cryonirsp-0.0.47 → dkist_processing_cryonirsp-0.0.49}/dkist_processing_cryonirsp/parsers/optical_density_filters.py

@@ -4,9 +4,7 @@ from typing import Hashable
 from dkist_processing_common.models.flower_pot import Stem
 from dkist_processing_common.models.flower_pot import Thorn
 
-from dkist_processing_cryonirsp.models.optical_density_filter_names import (
-    AllowableOpticalDensityFilterNames,
-)
+from dkist_processing_cryonirsp.models.exposure_conditions import AllowableOpticalDensityFilterNames
 from dkist_processing_cryonirsp.parsers.cryonirsp_l0_fits_access import CryonirspRampFitsAccess
 
 ALLOWABLE_OPTICAL_DENSITY_FILTERS = {f for f in AllowableOpticalDensityFilterNames}