dkist-processing-cryonirsp 0.0.62__tar.gz → 0.0.64__tar.gz

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/CHANGELOG.rst

@@ -1,3 +1,32 @@
+v0.0.64 (2024-06-03)
+====================
+
+Misc
+----
+
+- Resolve matplotlib version conflict (`#127 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/127>`__)
+- Upgrade the version of dkist-processing-common which brings along various major version upgrades to libraries associated with Pydantic 2. (`#128 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/128>`__)
+
+
+v0.0.63 (2024-05-20)
+====================
+
+Bugfixes
+--------
+
+- Polcal fit/modulation matrix quality metrics are now correctly rendered in quality report. (`#125 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/125>`__)
+- Correctly render L0 quality metrics (individual frame and dataset average and RMS values for dark, lamp, and solar frames).
+  Previously these had been missing from quality reports of polarimetric datasets. (`#125 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/125>`__)
+
+
+Misc
+----
+
+- Remove `CryoStemName.modstate` and `CryonirspTag.modstate`. Both of these already exist in `*-common` and can be used directly from there. (`#125 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/125>`__)
+- Update `dkist-processing-common` to v6.2.4. This fixes a bug that could cause the quality report to fail to render if
+  the demodulation matrices were fit with the (very old) "use_M12" fit mode. (`#126 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/126>`__)
+
+
 v0.0.62 (2024-05-16)
 ====================
 

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist_processing_cryonirsp
-Version: 0.0.62
+Version: 0.0.64
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/__init__.py

@@ -1,5 +1,5 @@
 """Init."""
-from dkist_service_configuration import logger  # first import to set logging.BasicConfig
+from dkist_service_configuration.logging import logger  # first import to set logging.BasicConfig
 from pkg_resources import DistributionNotFound
 from pkg_resources import get_distribution
 

dkist_processing_cryonirsp-0.0.64/dkist_processing_cryonirsp/config.py

@@ -0,0 +1,12 @@
+"""Configuration for the dkist-processing-cryonirsp package and the logging thereof."""
+from dkist_processing_common.config import DKISTProcessingCommonConfiguration
+
+
+class DKISTProcessingCryoNIRSPConfigurations(DKISTProcessingCommonConfiguration):
+    """Configurations custom to the dkist-processing-cryonirsp package."""
+
+    pass  # nothing custom yet
+
+
+dkist_processing_cryonirsp_configurations = DKISTProcessingCryoNIRSPConfigurations()
+dkist_processing_cryonirsp_configurations.log_configurations()

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/models/tags.py

@@ -3,7 +3,6 @@ from enum import Enum
 
 from dkist_processing_common.models.tags import StemName
 from dkist_processing_common.models.tags import Tag
-from dkist_processing_common.models.task_name import TaskName
 
 from dkist_processing_cryonirsp.models.exposure_conditions import ExposureConditions
 from dkist_processing_cryonirsp.models.task_name import CryonirspTaskName
@@ -18,7 +17,6 @@ class CryonirspStemName(str, Enum):
     curr_frame_in_ramp = "CURR_FRAME_IN_RAMP"
     time_obs = "TIME_OBS"
     meas_num = "MEAS_NUM"
-    modstate = "MODSTATE"
     map_scan = "MAP_SCAN"
     exposure_conditions = "EXPOSURE_CONDITIONS"
 
@@ -74,22 +72,6 @@ class CryonirspTag(Tag):
         """
         return cls.format_tag(CryonirspStemName.map_scan, map_scan)
 
-    @classmethod
-    def modstate(cls, modstate: int) -> str:
-        """
-        Tags by the current modstate number.
-
-        Parameters
-        ----------
-        modstate
-            The current scan step number
-
-        Returns
-        -------
-        The formatted tag string
-        """
-        return cls.format_tag(CryonirspStemName.modstate, modstate)
-
     @classmethod
     def linearized(cls) -> str:
         """

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/parsers/modstates.py

@@ -2,11 +2,11 @@
 from typing import Type
 
 from dkist_processing_common.models.flower_pot import SpilledDirt
+from dkist_processing_common.models.tags import StemName
 from dkist_processing_common.parsers.single_value_single_key_flower import (
     SingleValueSingleKeyFlower,
 )
 
-from dkist_processing_cryonirsp.models.tags import CryonirspStemName
 from dkist_processing_cryonirsp.parsers.cryonirsp_l0_fits_access import CryonirspL0FitsAccess
 
 
@@ -14,9 +14,7 @@ class ModstateNumberFlower(SingleValueSingleKeyFlower):
     """Flower for a modstate number."""
 
     def __init__(self):
-        super().__init__(
-            tag_stem_name=CryonirspStemName.modstate.value, metadata_key="modulator_state"
-        )
+        super().__init__(tag_stem_name=StemName.modstate.value, metadata_key="modulator_state")
 
     def setter(self, fits_obj: CryonirspL0FitsAccess) -> Type[SpilledDirt] | int:
         """
@@ -27,8 +25,6 @@ class ModstateNumberFlower(SingleValueSingleKeyFlower):
         fits_obj:
             A single FitsAccess object
         """
-        if fits_obj.ip_task_type != "observe" and fits_obj.ip_task_type != "polcal":
-            return SpilledDirt
         # Some intensity data incorrectly has modulator state = 0
         if getattr(fits_obj, self.metadata_key) == 0:
             return 1

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/assemble_movie.py

@@ -2,7 +2,7 @@
 import numpy as np
 from dkist_processing_common.codecs.fits import fits_array_decoder
 from dkist_processing_common.tasks import AssembleMovie
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 from PIL import ImageDraw
 from PIL.ImageFont import FreeTypeFont
 

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/bad_pixel_map.py

@@ -2,7 +2,7 @@
 import numpy as np
 import scipy.ndimage as spnd
 from dkist_processing_math.statistics import average_numpy_arrays
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.models.tags import CryonirspTag
 from dkist_processing_cryonirsp.models.task_name import CryonirspTaskName

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/beam_boundaries_base.py

@@ -4,7 +4,7 @@ from dataclasses import dataclass
 
 import numpy as np
 from dkist_processing_math.statistics import average_numpy_arrays
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 from largestinteriorrectangle import lir
 from skimage import filters
 from skimage.exposure import rescale_intensity

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/ci_beam_boundaries.py

@@ -1,7 +1,7 @@
 """Cryonirsp CI beam boundaries task."""
 import largestinteriorrectangle as lir
 import numpy as np
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.tasks.beam_boundaries_base import BeamBoundariesCalibrationBase
 from dkist_processing_cryonirsp.tasks.beam_boundaries_base import BeamBoundary

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/ci_science.py

@@ -1,7 +1,7 @@
 """Cryonirsp CI science calibration task."""
 from collections import defaultdict
 
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.models.exposure_conditions import ExposureConditions
 from dkist_processing_cryonirsp.models.tags import CryonirspTag

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/dark.py

@@ -1,7 +1,7 @@
 """CryoNIRSP dark calibration task."""
 from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_math.statistics import average_numpy_arrays
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.models.tags import CryonirspTag
 from dkist_processing_cryonirsp.tasks.cryonirsp_base import CryonirspTaskBase

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/gain.py

@@ -6,7 +6,7 @@ import numpy as np
 from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_math.arithmetic import subtract_array_from_arrays
 from dkist_processing_math.statistics import average_numpy_arrays
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.models.exposure_conditions import ExposureConditions
 from dkist_processing_cryonirsp.models.tags import CryonirspTag

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/instrument_polarization.py

@@ -13,7 +13,7 @@ from dkist_processing_math.transform.binning import resize_arrays
 from dkist_processing_pac.fitter.polcal_fitter import PolcalFitter
 from dkist_processing_pac.input_data.drawer import Drawer
 from dkist_processing_pac.input_data.dresser import Dresser
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.models.exposure_conditions import ExposureConditions
 from dkist_processing_cryonirsp.models.tags import CryonirspTag
@@ -23,6 +23,15 @@ from dkist_processing_cryonirsp.tasks.cryonirsp_base import CryonirspTaskBase
 __all__ = ["CIInstrumentPolarizationCalibration", "SPInstrumentPolarizationCalibration"]
 
 
+def generate_polcal_quality_label(arm: str, beam: int) -> str:
+    """
+    Make a quality label given an arm and beam.
+
+    Defined here so we don't have to remember what our labels are in the L1 output data task.
+    """
+    return f"{arm} Beam {beam}"
+
+
 class InstrumentPolarizationCalibrationBase(CryonirspTaskBase, ABC):
     """
     Base task class for instrument polarization for a CryoNIRSP calibration run.
@@ -402,7 +411,7 @@ class CIInstrumentPolarizationCalibration(InstrumentPolarizationCalibrationBase)
         """Record various quality metrics from PolCal fits."""
         self.quality_store_polcal_results(
             polcal_fitter=polcal_fitter,
-            label=f"CryoNIRSP CI beam {beam}",
+            label=generate_polcal_quality_label(arm="CI", beam=beam),
             bins_1=self.parameters.polcal_num_spatial_bins,
             bins_2=self.parameters.polcal_num_spatial_bins,
             bin_1_type="spatial",
@@ -421,7 +430,7 @@ class SPInstrumentPolarizationCalibration(InstrumentPolarizationCalibrationBase)
         """Record various quality metrics from PolCal fits."""
         self.quality_store_polcal_results(
             polcal_fitter=polcal_fitter,
-            label=f"CryoNIRSP SP beam {beam}",
+            label=generate_polcal_quality_label(arm="SP", beam=beam),
             bins_1=self.parameters.polcal_num_spatial_bins,
             bins_2=self.parameters.polcal_num_spectral_bins,
             bin_1_type="spatial",

dkist_processing_cryonirsp-0.0.64/dkist_processing_cryonirsp/tasks/l1_output_data.py

@@ -0,0 +1,44 @@
+"""Subclasses of AssembleQualityData that cause the correct polcal metrics to build."""
+from typing import Type
+
+from dkist_processing_common.models.constants import ConstantsBase
+from dkist_processing_common.tasks import AssembleQualityData
+
+__all__ = ["CIAssembleQualityData", "SPAssembleQualityData"]
+
+from dkist_processing_cryonirsp.models.constants import CryonirspConstants
+from dkist_processing_cryonirsp.tasks.instrument_polarization import generate_polcal_quality_label
+
+
+class CIAssembleQualityData(AssembleQualityData):
+    """Subclass just so that the polcal_label_list can be populated."""
+
+    @property
+    def constants_model_class(self) -> Type[CryonirspConstants]:
+        """Grab the Cryo constants so we can have the number of beams."""
+        return CryonirspConstants
+
+    @property
+    def polcal_label_list(self) -> list[str]:
+        """Return label(s) for Cryo CI."""
+        return [
+            generate_polcal_quality_label(arm="CI", beam=beam)
+            for beam in range(1, self.constants.num_beams + 1)
+        ]
+
+
+class SPAssembleQualityData(AssembleQualityData):
+    """Subclass just so that the polcal_label_list can be populated."""
+
+    @property
+    def constants_model_class(self) -> Type[CryonirspConstants]:
+        """Grab the Cryo constants so we can have the number of beams."""
+        return CryonirspConstants
+
+    @property
+    def polcal_label_list(self) -> list[str]:
+        """Return labels for beams 1 and 2."""
+        return [
+            generate_polcal_quality_label(arm="SP", beam=beam)
+            for beam in range(1, self.constants.num_beams + 1)
+        ]

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/linearity_correction.py

@@ -7,7 +7,7 @@ from astropy.io import fits
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.fits import fits_array_encoder
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 from numba import njit
 from numba import prange
 

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/make_movie_frames.py

@@ -7,7 +7,7 @@ from astropy.io import fits
 from astropy.visualization import ZScaleInterval
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.models.tags import CryonirspTag
 from dkist_processing_cryonirsp.parsers.cryonirsp_l1_fits_access import CryonirspL1FitsAccess

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/mixin/intermediate_frame.py

@@ -6,7 +6,7 @@ import numpy as np
 from astropy.io import fits
 from dkist_processing_common.codecs.fits import fits_hdu_decoder
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.models.exposure_conditions import ExposureConditions
 from dkist_processing_cryonirsp.models.tags import CryonirspTag

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/mixin/shift_measurements.py

@@ -4,7 +4,7 @@ from dataclasses import dataclass
 
 import numpy as np
 import skimage.registration as skir
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 SPATIAL = "SPATIAL"
 SPECTRAL = "SPECTRAL"

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/quality_metrics.py

@@ -9,7 +9,7 @@ from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.parsers.quality import L1QualityFitsAccess
 from dkist_processing_common.tasks import QualityL0Metrics
 from dkist_processing_common.tasks.mixin.quality import QualityMixin
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.models.constants import CryonirspConstants
 from dkist_processing_cryonirsp.models.tags import CryonirspTag

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/science_base.py

@@ -14,7 +14,7 @@ from dkist_processing_math.arithmetic import divide_arrays_by_array
 from dkist_processing_math.arithmetic import subtract_array_from_arrays
 from dkist_processing_math.statistics import average_numpy_arrays
 from dkist_processing_pac.optics.telescope import Telescope
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.models.exposure_conditions import ExposureConditions
 from dkist_processing_cryonirsp.models.tags import CryonirspTag

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/sp_beam_boundaries.py

@@ -2,7 +2,7 @@
 import math
 
 import numpy as np
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.tasks.beam_boundaries_base import BeamBoundariesCalibrationBase
 from dkist_processing_cryonirsp.tasks.beam_boundaries_base import BeamBoundary

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/sp_geometric.py

@@ -8,7 +8,7 @@ from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_math.arithmetic import divide_arrays_by_array
 from dkist_processing_math.arithmetic import subtract_array_from_arrays
 from dkist_processing_math.statistics import average_numpy_arrays
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 from scipy.optimize import minimize
 
 from dkist_processing_cryonirsp.models.tags import CryonirspTag

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/sp_science.py

@@ -4,7 +4,7 @@ from collections import defaultdict
 import numpy as np
 from astropy.io import fits
 from dkist_processing_math.statistics import average_numpy_arrays
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.models.exposure_conditions import ExposureConditions
 from dkist_processing_cryonirsp.models.tags import CryonirspTag

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/sp_solar_gain.py

@@ -5,7 +5,7 @@ from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_math.arithmetic import divide_arrays_by_array
 from dkist_processing_math.arithmetic import subtract_array_from_arrays
 from dkist_processing_math.statistics import average_numpy_arrays
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 from scipy import signal
 
 from dkist_processing_cryonirsp.models.exposure_conditions import ExposureConditions

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tasks/write_l1.py

@@ -9,7 +9,7 @@ import numpy as np
 from astropy.io import fits
 from dkist_processing_common.models.wavelength import WavelengthRange
 from dkist_processing_common.tasks import WriteL1Frame
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.models.constants import CryonirspConstants
 

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tests/conftest.py

@@ -284,7 +284,7 @@ def cryonirsp_testing_parameters_factory(
     class CryonirspTestingParameters:
         cryonirsp_polcal_num_spatial_bins: int = 1
         cryonirsp_polcal_num_spectral_bins: int = 1
-        cryonirsp_polcal_pac_fit_mode: str = "use_M12"
+        cryonirsp_polcal_pac_fit_mode: str = "use_M12_I_sys_per_step"
         cryonirsp_polcal_pac_init_set: str = "OCCal_VIS"
         cryonirsp_geo_upsample_factor: int = 100
         cryonirsp_geo_max_shift: int = 80

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tests/header_models.py

@@ -372,10 +372,12 @@ class ModulatedLampGainHeaders(SimpleModulatedHeaders):
         exposure_condition: ExposureConditions,
         start_date: str = "2023-01-01T01:23:45",
         modstate_length_sec: float = 0.5,
+        num_modstates: int = 1,
+        modstate: int = 1,
     ):
         super().__init__(
-            num_modstates=1,
-            modstate=1,
+            num_modstates=num_modstates,
+            modstate=modstate,
             array_shape=array_shape,
             task="gain",
             exposure_condition=exposure_condition,
@@ -403,10 +405,12 @@ class ModulatedSolarGainHeaders(SimpleModulatedHeaders):
         exposure_condition: ExposureConditions,
         start_date: str = "2023-01-01T01:23:45",
         modstate_length_sec: float = 0.5,
+        num_modstates: int = 1,
+        modstate: int = 1,
     ):
         super().__init__(
-            num_modstates=1,
-            modstate=1,
+            num_modstates=num_modstates,
+            modstate=modstate,
             array_shape=array_shape,
             task="gain",
             exposure_condition=exposure_condition,
@@ -436,10 +440,12 @@ class ModulatedDarkHeaders(SimpleModulatedHeaders):
         exposure_condition: ExposureConditions,
         start_date: str = "2023-01-01T01:23:45",
         modstate_length_sec: float = 0.5,
+        num_modstates: int = 1,
+        modstate: int = 1,
     ):
         super().__init__(
-            num_modstates=1,
-            modstate=1,
+            num_modstates=num_modstates,
+            modstate=modstate,
             array_shape=array_shape,
             task="dark",
             exposure_condition=exposure_condition,

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tests/local_trial_workflows/l0_cals_only.py

@@ -9,7 +9,7 @@ from astropy.io import fits
 from dkist_processing_common.manual import ManualProcessing
 from dkist_processing_common.models.constants import BudName
 from dkist_processing_common.tasks import WorkflowTaskBase
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.models.constants import CryonirspBudName
 from dkist_processing_cryonirsp.models.tags import CryonirspTag

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tests/local_trial_workflows/l0_to_l1.py

@@ -11,7 +11,7 @@ from dkist_processing_common.tasks import CreateTrialAsdf
 from dkist_processing_common.tasks import CreateTrialQualityReport
 from dkist_processing_common.tasks import QualityL1Metrics
 from dkist_processing_common.tasks import WorkflowTaskBase
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.models.constants import CryonirspBudName
 from dkist_processing_cryonirsp.models.tags import CryonirspTag
@@ -406,12 +406,15 @@ def main(
             )
         else:
             raise ValueError(f"Did not recognize {arm_id = }")
+
         manual_processing_run.run_task(task=QualityL1Metrics)
         manual_processing_run.run_task(task=CryonirspL1QualityMetrics)
+
         if arm_id == "SP":
             manual_processing_run.run_task(task=SPAssembleQualityData)
         elif arm_id == "CI":
             manual_processing_run.run_task(task=CIAssembleQualityData)
+
         manual_processing_run.run_task(task=ValidateL1Output)
 
         if transfer_trial_data:

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tests/local_trial_workflows/local_trial_helpers.py

@@ -11,7 +11,7 @@ from dkist_header_validator import spec122_validator
 from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_common.tasks import WorkflowTaskBase
 from dkist_processing_common.tasks.mixin.globus import GlobusTransferItem
-from dkist_service_configuration import logger
+from dkist_service_configuration.logging import logger
 
 from dkist_processing_cryonirsp.models.tags import CryonirspTag
 from dkist_processing_cryonirsp.models.task_name import CryonirspTaskName
@@ -171,15 +171,24 @@ class SaveTaskTags(WorkflowTaskBase):
     def relative_save_file(self) -> str:
         return "default_sav.asdf"
 
+    @property
+    def tag_lists_to_save(self) -> list[list[str]] | list[str]:
+        return [[CryonirspTag.task(self.task_str), CryonirspTag.intermediate()]]
+
     def run(self):
         file_tag_dict = dict()
-        path_list = self.read(tags=[CryonirspTag.task(self.task_str), CryonirspTag.intermediate()])
-        save_dir = self.scratch.workflow_base_path / Path(self.relative_save_file).stem
-        save_dir.mkdir(exist_ok=True)
-        for p in path_list:
-            copied_path = shutil.copy(str(p), save_dir)
-            tags = self.tags(p)
-            file_tag_dict[copied_path] = tags
+        tag_list_list = self.tag_lists_to_save
+        if isinstance(tag_list_list[0], str):
+            tag_list_list = [tag_list_list]
+
+        for tags_to_save in tag_list_list:
+            path_list = self.read(tags=tags_to_save)
+            save_dir = self.scratch.workflow_base_path / Path(self.relative_save_file).stem
+            save_dir.mkdir(exist_ok=True)
+            for p in path_list:
+                copied_path = shutil.copy(str(p), save_dir)
+                tags = self.tags(p)
+                file_tag_dict[copied_path] = tags
 
         full_save_file = self.scratch.workflow_base_path / self.relative_save_file
         tree = {"file_tag_dict": file_tag_dict}
@@ -216,6 +225,14 @@ class SaveGeometricCal(WorkflowTaskBase):
                 tags=[CryonirspTag.task_geometric_sepectral_shifts(), CryonirspTag.intermediate()]
             )
         )
+        path_list += list(
+            self.read(
+                tags=[
+                    CryonirspTag.quality("TASK_TYPES"),
+                    CryonirspTag.workflow_task("SPGeometricCalibration"),
+                ]
+            )
+        )
         save_dir = self.scratch.workflow_base_path / Path(relative_save_file).stem
         save_dir.mkdir(exist_ok=True)
         for p in path_list:
@@ -277,6 +294,12 @@ class SaveDarkCal(SaveTaskTags):
     def relative_save_file(self) -> str:
         return "dark_cal.asdf"
 
+    @property
+    def tag_lists_to_save(self) -> list[list[str]]:
+        return super().tag_lists_to_save + [
+            [CryonirspTag.quality("TASK_TYPES"), CryonirspTag.workflow_task("DarkCalibration")]
+        ]
+
 
 class LoadDarkCal(LoadTaskTags):
     @property
@@ -293,6 +316,12 @@ class SaveLampCal(SaveTaskTags):
     def relative_save_file(self) -> str:
         return "lamp_cal.asdf"
 
+    @property
+    def tag_lists_to_save(self) -> list[list[str]]:
+        return super().tag_lists_to_save + [
+            [CryonirspTag.quality("TASK_TYPES"), CryonirspTag.workflow_task("LampGainCalibration")],
+        ]
+
 
 class LoadLampCal(LoadTaskTags):
     @property
@@ -309,6 +338,19 @@ class SaveSolarCal(SaveTaskTags):
     def relative_save_file(self) -> str:
         return "solar_cal.asdf"
 
+    @property
+    def tag_lists_to_save(self) -> list[list[str]]:
+        return super().tag_lists_to_save + [
+            [
+                CryonirspTag.quality("TASK_TYPES"),
+                CryonirspTag.workflow_task("CISolarGainCalibration"),
+            ],
+            [
+                CryonirspTag.quality("TASK_TYPES"),
+                CryonirspTag.workflow_task("SPSolarGainCalibration"),
+            ],
+        ]
+
 
 class LoadSolarCal(LoadTaskTags):
     @property
@@ -325,6 +367,19 @@ class SaveInstPolCal(SaveTaskTags):
     def relative_save_file(self) -> str:
         return "inst_pol_cal.asdf"
 
+    @property
+    def tag_lists_to_save(self) -> list[list[str]]:
+        return super().tag_lists_to_save + [
+            [
+                CryonirspTag.quality("TASK_TYPES"),
+                CryonirspTag.workflow_task("CIInstrumentPolarizationCalibration"),
+            ],
+            [
+                CryonirspTag.quality("TASK_TYPES"),
+                CryonirspTag.workflow_task("SPInstrumentPolarizationCalibration"),
+            ],
+        ]
+
 
 class LoadInstPolCal(LoadTaskTags):
     @property

{dkist_processing_cryonirsp-0.0.62 → dkist_processing_cryonirsp-0.0.64}/dkist_processing_cryonirsp/tests/test_assemble_qualilty.py

@@ -21,8 +21,13 @@ from dkist_processing_cryonirsp.tests.conftest import CryonirspConstantsDb
 
 
 @pytest.fixture(scope="function", params=["CI", "SP"])
-def cryo_assemble_quality_data_task(tmp_path, recipe_run_id, request) -> AssembleQualityData:
+def cryo_assemble_quality_data_task(
+    tmp_path, recipe_run_id, init_cryonirsp_constants_db, request
+) -> AssembleQualityData:
     arm_id = request.param
+    init_cryonirsp_constants_db(
+        recipe_run_id=recipe_run_id, constants_obj=CryonirspConstantsDb(ARM_ID=arm_id)
+    )
     if arm_id == "CI":
         with CIAssembleQualityData(
             recipe_run_id=recipe_run_id,
@@ -228,7 +233,7 @@ def quality_metrics(dataframe_json) -> list[Metric]:
                 "bin_1_str": "bin1",
                 "bin_2_str": "bin2",
                 "total_bins": 100,
-                "num_steps": 2,
+                "num_varied_I_sys": 2,
             },
             ["QUALITY_POLCAL_LOCAL_PAR_VALS", "QUALITY_TASK_CI BEAM 1"],
         ),