dkist-processing-cryonirsp 1.1.1rc1__tar.gz → 1.2.0__tar.gz

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/CHANGELOG.rst

@@ -1,3 +1,42 @@
+v1.2.0 (2024-09-23)
+===================
+
+Features
+--------
+
+- Reorder task dependencies in workflows. Movie and L1 quality tasks are no longer dependent on the presence of OUTPUT
+  frames and thus can be run in parallel with the `WriteL1` task. (`#152 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/152>`__)
+
+
+Misc
+----
+
+- Use CALIBRATED instead of OUTPUT frames in post-science movie and quality tasks. This doesn't change their output at all (the arrays are the same), but
+  it's necessary for `dkist-processing-common >= 10.0.0` that will break using OUTPUT frames. (`#151 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/151>`__)
+- Remove `AssembleCryonirspMovie` as workflow dependency on `SubmitDatasetMetadata`. This dependency has been unnecessary
+  since the introduction of `SubmitDatasetMetadata` in v0.0.60. (`#151 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/151>`__)
+
+
+v1.1.2 (2024-09-19)
+===================
+
+Misc
+----
+
+- Bump `dkist-quality` to v1.1.1. This fixes raincloud plot rendering in trial workflows. (`#152 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/152>`__)
+
+
+v1.1.1 (2024-09-18)
+===================
+
+Bugfixes
+--------
+
+- Add validation in linearity_correction task to ensure that the value of NUM_FRAMES_IN_RAMP is the same
+  across all frames in a ramp, and that the value of NUM_FRAMES_IN_RAMP actually matches the number of frames found. (`#147 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/147>`__)
+- When writing linearized frames, use a fixed tag list as opposed to one derived from L0 data to alleviate load on redis. (`#148 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/148>`__)
+
+
 v1.1.0 (2024-09-10)
 ===================
 

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist_processing_cryonirsp
-Version: 1.1.1rc1
+Version: 1.2.0
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/bitbucket-pipelines.yml

@@ -47,6 +47,7 @@ definitions:
         caches:
           - pip
         name: Test
+        size: 4x
         script:
           - pip install -U pip
           - pip install .[test]

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/dkist_processing_cryonirsp/tasks/assemble_movie.py

@@ -119,6 +119,7 @@ class AssembleCryonirspMovieBase(AssembleMovie):
         return line
 
 
+# See note below on `SPAssembleCryonirspMovie`
 class AssembleCryonirspMovie(AssembleCryonirspMovieBase):
     """
     Assemble all CryoNIRSP CI movie frames (tagged with CryonirspTag.movie_frame()) into an mp4 movie file.
@@ -162,6 +163,10 @@ class AssembleCryonirspMovie(AssembleCryonirspMovieBase):
         return tags
 
 
+# NOTE:
+# This task isn't used right now because the SP movies need some better handling of the wavelength dimension.
+# For the time being both arms use the `MakeCryonirspMovieFrames` task above.
+# See PR #91 for more information.
 class SPAssembleCryonirspMovie(AssembleCryonirspMovieBase):
     """
     Assemble all CryoNIRSP SP movie frames (tagged with CryonirspTag.movie_frame()) into an mp4 movie file.

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/dkist_processing_cryonirsp/tasks/make_movie_frames.py

@@ -67,7 +67,7 @@ class MakeCryonirspMovieFramesBase(CryonirspTaskBase, ABC):
             self.read(
                 tags=[
                     CryonirspTag.frame(),
-                    CryonirspTag.output(),
+                    CryonirspTag.calibrated(),
                     CryonirspTag.stokes(stokes_state),
                     CryonirspTag.meas_num(meas_num),
                     CryonirspTag.map_scan(map_scan),
@@ -81,6 +81,7 @@ class MakeCryonirspMovieFramesBase(CryonirspTaskBase, ABC):
         return header
 
 
+# See note below on `SPMakeCryonirspMovieFrames`
 class MakeCryonirspMovieFrames(MakeCryonirspMovieFramesBase):
     """Make CryoNIRSP movie frames for the Context Imager and tag with CryonirspTag.movie_frame()."""
 
@@ -150,7 +151,7 @@ class MakeCryonirspMovieFrames(MakeCryonirspMovieFramesBase):
             self.read(
                 tags=[
                     CryonirspTag.frame(),
-                    CryonirspTag.output(),
+                    CryonirspTag.calibrated(),
                     CryonirspTag.stokes(stokes_state),
                     CryonirspTag.meas_num(meas_num),
                     CryonirspTag.map_scan(map_scan),
@@ -164,6 +165,10 @@ class MakeCryonirspMovieFrames(MakeCryonirspMovieFramesBase):
         return movie_frame
 
 
+# NOTE:
+# This task isn't used right now because the SP movies need some better handling of the wavelength dimension.
+# For the time being both arms use the `MakeCryonirspMovieFrames` task above.
+# See PR #91 for more information.
 class SPMakeCryonirspMovieFrames(MakeCryonirspMovieFramesBase):
     """Make CryoNIRSP movie frames for the Spectropolarimeter and tag with CryonirspTag.movie_frame()."""
 
@@ -249,7 +254,7 @@ class SPMakeCryonirspMovieFrames(MakeCryonirspMovieFramesBase):
             self.read(
                 tags=[
                     CryonirspTag.frame(),
-                    CryonirspTag.output(),
+                    CryonirspTag.calibrated(),
                     CryonirspTag.stokes(stokes_state),
                     CryonirspTag.meas_num(meas_num),
                     CryonirspTag.map_scan(map_scan),
@@ -275,7 +280,7 @@ class SPMakeCryonirspMovieFrames(MakeCryonirspMovieFramesBase):
                 self.read(
                     tags=[
                         CryonirspTag.frame(),
-                        CryonirspTag.output(),
+                        CryonirspTag.calibrated(),
                         CryonirspTag.stokes(stokes_state),
                         CryonirspTag.meas_num(meas_num),
                         CryonirspTag.map_scan(map_scan),

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/dkist_processing_cryonirsp/tasks/quality_metrics.py

@@ -205,7 +205,7 @@ class CryonirspL1QualityMetrics(CryonirspTaskBase, QualityMixin):
             frame = next(
                 self.read(
                     tags=[
-                        CryonirspTag.output(),
+                        CryonirspTag.calibrated(),
                         CryonirspTag.frame(),
                         CryonirspTag.scan_step(step),
                         CryonirspTag.map_scan(map_scan),
@@ -234,7 +234,7 @@ class CryonirspL1QualityMetrics(CryonirspTaskBase, QualityMixin):
         frame: L1QualityFitsAccess = next(
             self.read(
                 tags=[
-                    CryonirspTag.output(),
+                    CryonirspTag.calibrated(),
                     CryonirspTag.frame(),
                     CryonirspTag.scan_step(step),
                     CryonirspTag.map_scan(map_scan),
@@ -255,7 +255,7 @@ class CryonirspL1QualityMetrics(CryonirspTaskBase, QualityMixin):
 
     def compile_noise_data(self, stokes: str) -> _QualityData:
         """Compile lists of noise values and their observation times."""
-        tags = [CryonirspTag.output(), CryonirspTag.frame(), CryonirspTag.stokes(stokes)]
+        tags = [CryonirspTag.calibrated(), CryonirspTag.frame(), CryonirspTag.stokes(stokes)]
         frames: Generator[L1QualityFitsAccess, None, None] = self.read(
             tags=tags,
             decoder=fits_access_decoder,

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/dkist_processing_cryonirsp/tests/local_trial_workflows/local_trial_helpers.py

@@ -407,6 +407,11 @@ class SaveInstPolCal(SaveTaskTags):
                 CryonirspTag.quality("TASK_TYPES"),
                 CryonirspTag.workflow_task("SPInstrumentPolarizationCalibration"),
             ],
+            [CryonirspTag.quality("POLCAL_CONSTANT_PAR_VALS")],
+            [CryonirspTag.quality("POLCAL_GLOBAL_PAR_VALS")],
+            [CryonirspTag.quality("POLCAL_LOCAL_PAR_VALS")],
+            [CryonirspTag.quality("POLCAL_FIT_RESIDUALS")],
+            [CryonirspTag.quality("POLCAL_EFFICIENCY")],
         ]
 
 

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/dkist_processing_cryonirsp/tests/test_make_movie_frames.py

@@ -1,6 +1,5 @@
 from datetime import datetime
 
-import numpy as np
 import pytest
 from astropy.io import fits
 from dkist_header_validator import spec122_validator
@@ -77,7 +76,7 @@ def movie_frames_task(tmp_path, recipe_run_id, init_cryonirsp_constants_db, requ
                         task.write(
                             data=hdul,
                             tags=[
-                                CryonirspTag.output(),
+                                CryonirspTag.calibrated(),
                                 CryonirspTag.frame(),
                                 CryonirspTag.map_scan(map_scan),
                                 CryonirspTag.scan_step(scan_step),

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/dkist_processing_cryonirsp/tests/test_quality.py

@@ -115,7 +115,7 @@ def cryo_l1_quality_task(tmp_path, recipe_run_id, init_cryonirsp_constants_db):
                     task.write(
                         data=hdul,
                         tags=[
-                            CryonirspTag.output(),
+                            CryonirspTag.calibrated(),
                             CryonirspTag.frame(),
                             CryonirspTag.stokes(stokes_param),
                             CryonirspTag.scan_step(scan_step),

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/dkist_processing_cryonirsp/tests/test_sp_make_movie_frames.py

@@ -81,7 +81,7 @@ def sp_movie_frames_task(tmp_path, recipe_run_id, init_cryonirsp_constants_db, r
                         task.write(
                             data=hdul,
                             tags=[
-                                CryonirspTag.output(),
+                                CryonirspTag.calibrated(),
                                 CryonirspTag.frame(),
                                 CryonirspTag.map_scan(map_scan),
                                 CryonirspTag.scan_step(scan_step),

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/dkist_processing_cryonirsp/tests/test_write_l1.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from pathlib import Path
 
 import numpy as np
 import pytest
@@ -176,56 +177,54 @@ def test_write_l1_frame(write_l1_task, mocker, arm_id, num_stokes_params, num_ma
     task()
 
     for stokes_param in stokes_params:
-        for map_scan in range(1, task.constants.num_map_scans + 1):
-            for scan_step in range(1, task.constants.num_scan_steps + 1):
-                for meas_num in range(1, task.constants.num_meas + 1):
-                    common_tags = [
-                        CryonirspTag.frame(),
-                        CryonirspTag.stokes(stokes_param),
-                        CryonirspTag.map_scan(map_scan),
-                        CryonirspTag.scan_step(scan_step),
-                        CryonirspTag.meas_num(meas_num),
-                    ]
-                    output_files = list(task.read(tags=common_tags + [CryonirspTag.output()]))
-                    calibrated_files = list(
-                        task.read(tags=common_tags + [CryonirspTag.calibrated()])
-                    )
-                    assert len(output_files) == 1
-                    output_file = output_files.pop()
-                    assert output_file.exists
-                    assert output_file.name.startswith(f"CRYO-NIRSP_{arm_id}_")
-                    assert spec214_validator.validate(output_file, extra=False)
-                    hdu_list = fits.open(output_file)
-                    header = hdu_list[1].header
-                    assert len(hdu_list) == 2  # Primary, CompImage
-                    assert type(hdu_list[0]) is fits.PrimaryHDU
-                    assert type(hdu_list[1]) is fits.CompImageHDU
-                    axis_num = 1
-                    if arm_id == "CI":
-                        longitude_axis_tests(axis_num, header)
-                        ci_spatial_axis_correction_tests(axis_num, original_headers, header)
-                    else:
-                        spectral_axis_tests(axis_num, header)
-                        sp_spatial_axis_correction_tests(axis_num, original_headers, header)
-                        sp_spectral_axis_correction_tests(original_headers, header)
-                    axis_num += 1
-                    latitude_axis_tests(axis_num, arm_id, header)
-                    axis_num += 1
-                    if task.constants.num_meas > 1:
-                        measurement_axis_tests(axis_num, header, task)
-                        axis_num += 1
-                    scan_step_axis_tests(axis_num, arm_id, header, task)
-                    if task.constants.num_map_scans > 1:
-                        axis_num += 1
-                        map_scan_tests(axis_num, header, task)
-                    if task.constants.correct_for_polarization:
-                        axis_num += 1
-                        stokes_axis_tests(axis_num, header)
-
-                    # Other general tests
-                    general_header_tests(axis_num, header, task, arm_id, calibrated_files)
-
-                    axis_flip_tests(original_headers, header, arm_id)
+        common_tags = [
+            CryonirspTag.frame(),
+            CryonirspTag.stokes(stokes_param),
+        ]
+        output_files = list(task.read(tags=common_tags + [CryonirspTag.output()]))
+        calibrated_files = list(task.read(tags=common_tags + [CryonirspTag.calibrated()]))
+        assert (
+            len(output_files)
+            == task.constants.num_map_scans
+            * task.constants.num_scan_steps
+            * task.constants.num_meas
+        )
+        preserve_date_tests(output_files, calibrated_files)
+        for output_file in output_files:
+            assert output_file.exists
+            assert output_file.name.startswith(f"CRYO-NIRSP_{arm_id}_")
+            assert spec214_validator.validate(output_file, extra=False)
+            hdu_list = fits.open(output_file)
+            header = hdu_list[1].header
+            assert len(hdu_list) == 2  # Primary, CompImage
+            assert type(hdu_list[0]) is fits.PrimaryHDU
+            assert type(hdu_list[1]) is fits.CompImageHDU
+            axis_num = 1
+            if arm_id == "CI":
+                longitude_axis_tests(axis_num, header)
+                ci_spatial_axis_correction_tests(axis_num, original_headers, header)
+            else:
+                spectral_axis_tests(axis_num, header)
+                sp_spatial_axis_correction_tests(axis_num, original_headers, header)
+                sp_spectral_axis_correction_tests(original_headers, header)
+            axis_num += 1
+            latitude_axis_tests(axis_num, arm_id, header)
+            axis_num += 1
+            if task.constants.num_meas > 1:
+                measurement_axis_tests(axis_num, header, task)
+                axis_num += 1
+            scan_step_axis_tests(axis_num, arm_id, header, task)
+            if task.constants.num_map_scans > 1:
+                axis_num += 1
+                map_scan_tests(axis_num, header, task)
+            if task.constants.correct_for_polarization:
+                axis_num += 1
+                stokes_axis_tests(axis_num, header)
+
+            # Other general tests
+            general_header_tests(axis_num, header, task, arm_id)
+
+            axis_flip_tests(original_headers, header, arm_id)
 
 
 def sp_spatial_axis_correction_tests(
@@ -382,12 +381,24 @@ def axis_flip_tests(original_header: fits.Header, header: fits.Header, arm_id: s
         pass
 
 
+def preserve_date_tests(output_files: list[Path], calibrated_files: list[Path]) -> None:
+    # Make sure we didn't overwrite pre-computed DATE-BEG and DATE-END keys
+    cal_headers = [fits.getheader(f) for f in calibrated_files]
+    output_headers = [fits.getheader(f, ext=1) for f in output_files]
+
+    assert sorted([h["DATE-BEG"] for h in cal_headers]) == sorted(
+        [h["DATE-BEG"] for h in output_headers]
+    )
+    assert sorted([h["DATE-END"] for h in cal_headers]) == sorted(
+        [h["DATE-END"] for h in output_headers]
+    )
+
+
 def general_header_tests(
     axis_num: int,
     header: fits.Header,
     task: CryonirspWriteL1Frame,
     arm_id: str,
-    calibrated_files: list,
 ):
     # Other general tests
     assert header["DAAXES"] == 2
@@ -404,11 +415,6 @@ def general_header_tests(
         header["CAL_URL"]
         == f"{task.docs_base_url}/projects/{task.constants.instrument.lower()}/en/v{calvers}/{task.workflow_name}.html"
     )
-    cal_header = fits.open(calibrated_files.pop())[0].header
-
-    # Make sure we didn't overwrite pre-computed DATE-BEG and DATE-END keys
-    assert header["DATE-BEG"] == cal_header["DATE-BEG"]
-    assert header["DATE-END"] == cal_header["DATE-END"]
     date_avg = (
         (Time(header["DATE-END"], precision=6) - Time(header["DATE-BEG"], precision=6)) / 2
         + Time(header["DATE-BEG"], precision=6)

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/dkist_processing_cryonirsp/workflows/ci_l0_processing.py

@@ -35,6 +35,8 @@ l0_pipeline = Workflow(
     workflow_package=__package__,
 )
 l0_pipeline.add_node(task=TransferL0Data, upstreams=None)
+
+# Science flow
 l0_pipeline.add_node(task=ParseL0CryonirspRampData, upstreams=TransferL0Data)
 l0_pipeline.add_node(
     task=LinearityCorrection,
@@ -42,7 +44,6 @@ l0_pipeline.add_node(
     upstreams=ParseL0CryonirspRampData,
 )
 l0_pipeline.add_node(task=ParseL0CryonirspCILinearizedData, upstreams=LinearityCorrection)
-l0_pipeline.add_node(task=CryonirspL0QualityMetrics, upstreams=ParseL0CryonirspCILinearizedData)
 l0_pipeline.add_node(task=BadPixelMapCalibration, upstreams=ParseL0CryonirspCILinearizedData)
 l0_pipeline.add_node(task=CIBeamBoundariesCalibration, upstreams=BadPixelMapCalibration)
 l0_pipeline.add_node(task=DarkCalibration, upstreams=CIBeamBoundariesCalibration)
@@ -51,19 +52,26 @@ l0_pipeline.add_node(task=CISolarGainCalibration, upstreams=LampGainCalibration)
 l0_pipeline.add_node(task=CIInstrumentPolarizationCalibration, upstreams=CISolarGainCalibration)
 l0_pipeline.add_node(task=CIScienceCalibration, upstreams=CIInstrumentPolarizationCalibration)
 l0_pipeline.add_node(task=CIWriteL1Frame, upstreams=CIScienceCalibration)
-l0_pipeline.add_node(task=QualityL1Metrics, upstreams=CIWriteL1Frame)
-l0_pipeline.add_node(task=CryonirspL1QualityMetrics, upstreams=CIWriteL1Frame)
+
+# Movie flow
+l0_pipeline.add_node(task=MakeCryonirspMovieFrames, upstreams=CIScienceCalibration)
+l0_pipeline.add_node(task=AssembleCryonirspMovie, upstreams=MakeCryonirspMovieFrames)
+
+# Quality flow
+l0_pipeline.add_node(task=CryonirspL0QualityMetrics, upstreams=ParseL0CryonirspCILinearizedData)
+l0_pipeline.add_node(task=QualityL1Metrics, upstreams=CIScienceCalibration)
+l0_pipeline.add_node(task=CryonirspL1QualityMetrics, upstreams=CIScienceCalibration)
 l0_pipeline.add_node(
     task=CIAssembleQualityData,
     upstreams=[CryonirspL0QualityMetrics, QualityL1Metrics, CryonirspL1QualityMetrics],
 )
-l0_pipeline.add_node(task=MakeCryonirspMovieFrames, upstreams=CIWriteL1Frame)
-l0_pipeline.add_node(task=AssembleCryonirspMovie, upstreams=MakeCryonirspMovieFrames)
-l0_pipeline.add_node(
-    task=SubmitDatasetMetadata, upstreams=[AssembleCryonirspMovie, CIAssembleQualityData]
-)
-l0_pipeline.add_node(task=TransferL1Data, upstreams=AssembleCryonirspMovie)
+
+# Output flow
+l0_pipeline.add_node(task=SubmitDatasetMetadata, upstreams=[CIWriteL1Frame, CIAssembleQualityData])
+l0_pipeline.add_node(task=TransferL1Data, upstreams=[CIWriteL1Frame, AssembleCryonirspMovie])
 l0_pipeline.add_node(
     task=PublishCatalogAndQualityMessages, upstreams=[SubmitDatasetMetadata, TransferL1Data]
 )
+
+# goodbye
 l0_pipeline.add_node(task=Teardown, upstreams=PublishCatalogAndQualityMessages)

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/dkist_processing_cryonirsp/workflows/sp_l0_processing.py

@@ -37,6 +37,8 @@ l0_pipeline = Workflow(
     workflow_package=__package__,
 )
 l0_pipeline.add_node(task=TransferL0Data, upstreams=None)
+
+# Science flow
 l0_pipeline.add_node(task=ParseL0CryonirspRampData, upstreams=TransferL0Data)
 l0_pipeline.add_node(
     task=LinearityCorrection,
@@ -44,7 +46,6 @@ l0_pipeline.add_node(
     upstreams=ParseL0CryonirspRampData,
 )
 l0_pipeline.add_node(task=ParseL0CryonirspSPLinearizedData, upstreams=LinearityCorrection)
-l0_pipeline.add_node(task=CryonirspL0QualityMetrics, upstreams=ParseL0CryonirspSPLinearizedData)
 l0_pipeline.add_node(task=BadPixelMapCalibration, upstreams=ParseL0CryonirspSPLinearizedData)
 l0_pipeline.add_node(task=SPBeamBoundariesCalibration, upstreams=BadPixelMapCalibration)
 l0_pipeline.add_node(task=DarkCalibration, upstreams=SPBeamBoundariesCalibration)
@@ -58,19 +59,26 @@ l0_pipeline.add_node(
     upstreams=[SPInstrumentPolarizationCalibration, SPDispersionAxisCorrection],
 )
 l0_pipeline.add_node(task=SPWriteL1Frame, upstreams=SPScienceCalibration)
-l0_pipeline.add_node(task=QualityL1Metrics, upstreams=SPWriteL1Frame)
-l0_pipeline.add_node(task=CryonirspL1QualityMetrics, upstreams=SPWriteL1Frame)
+
+# Movie flow
+l0_pipeline.add_node(task=MakeCryonirspMovieFrames, upstreams=SPScienceCalibration)
+l0_pipeline.add_node(task=AssembleCryonirspMovie, upstreams=MakeCryonirspMovieFrames)
+
+# Quality flow
+l0_pipeline.add_node(task=CryonirspL0QualityMetrics, upstreams=ParseL0CryonirspSPLinearizedData)
+l0_pipeline.add_node(task=QualityL1Metrics, upstreams=SPScienceCalibration)
+l0_pipeline.add_node(task=CryonirspL1QualityMetrics, upstreams=SPScienceCalibration)
 l0_pipeline.add_node(
     task=SPAssembleQualityData,
     upstreams=[CryonirspL0QualityMetrics, QualityL1Metrics, CryonirspL1QualityMetrics],
 )
-l0_pipeline.add_node(task=MakeCryonirspMovieFrames, upstreams=SPWriteL1Frame)
-l0_pipeline.add_node(task=AssembleCryonirspMovie, upstreams=MakeCryonirspMovieFrames)
-l0_pipeline.add_node(
-    task=SubmitDatasetMetadata, upstreams=[AssembleCryonirspMovie, SPAssembleQualityData]
-)
-l0_pipeline.add_node(task=TransferL1Data, upstreams=AssembleCryonirspMovie)
+
+# Output flow
+l0_pipeline.add_node(task=SubmitDatasetMetadata, upstreams=[SPWriteL1Frame, SPAssembleQualityData])
+l0_pipeline.add_node(task=TransferL1Data, upstreams=[SPWriteL1Frame, AssembleCryonirspMovie])
 l0_pipeline.add_node(
     task=PublishCatalogAndQualityMessages, upstreams=[SubmitDatasetMetadata, TransferL1Data]
 )
+
+# goodbye
 l0_pipeline.add_node(task=Teardown, upstreams=PublishCatalogAndQualityMessages)

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/dkist_processing_cryonirsp/workflows/trial_workflows.py

@@ -47,6 +47,8 @@ full_trial_ci_pipeline = Workflow(
     workflow_package=__package__,
 )
 full_trial_ci_pipeline.add_node(task=TransferL0Data, upstreams=None)
+
+# Science flow
 full_trial_ci_pipeline.add_node(task=ParseL0CryonirspRampData, upstreams=TransferL0Data)
 full_trial_ci_pipeline.add_node(
     task=LinearityCorrection,
@@ -56,9 +58,6 @@ full_trial_ci_pipeline.add_node(
 full_trial_ci_pipeline.add_node(
     task=ParseL0CryonirspCILinearizedData, upstreams=LinearityCorrection
 )
-full_trial_ci_pipeline.add_node(
-    task=CryonirspL0QualityMetrics, upstreams=ParseL0CryonirspCILinearizedData
-)
 full_trial_ci_pipeline.add_node(
     task=BadPixelMapCalibration, upstreams=ParseL0CryonirspCILinearizedData
 )
@@ -73,14 +72,23 @@ full_trial_ci_pipeline.add_node(
     task=CIScienceCalibration, upstreams=CIInstrumentPolarizationCalibration
 )
 full_trial_ci_pipeline.add_node(task=CIWriteL1Frame, upstreams=CIScienceCalibration)
-full_trial_ci_pipeline.add_node(task=QualityL1Metrics, upstreams=CIWriteL1Frame)
-full_trial_ci_pipeline.add_node(task=CryonirspL1QualityMetrics, upstreams=CIWriteL1Frame)
+
+# Movie flow
+full_trial_ci_pipeline.add_node(task=MakeCryonirspMovieFrames, upstreams=CIScienceCalibration)
+full_trial_ci_pipeline.add_node(task=AssembleCryonirspMovie, upstreams=MakeCryonirspMovieFrames)
+
+# Quality flow
+full_trial_ci_pipeline.add_node(
+    task=CryonirspL0QualityMetrics, upstreams=ParseL0CryonirspCILinearizedData
+)
+full_trial_ci_pipeline.add_node(task=QualityL1Metrics, upstreams=CIScienceCalibration)
+full_trial_ci_pipeline.add_node(task=CryonirspL1QualityMetrics, upstreams=CIScienceCalibration)
 full_trial_ci_pipeline.add_node(
     task=CIAssembleQualityData,
     upstreams=[CryonirspL0QualityMetrics, QualityL1Metrics, CryonirspL1QualityMetrics],
 )
-full_trial_ci_pipeline.add_node(task=MakeCryonirspMovieFrames, upstreams=CIWriteL1Frame)
-full_trial_ci_pipeline.add_node(task=AssembleCryonirspMovie, upstreams=MakeCryonirspMovieFrames)
+
+# Trial data generation
 full_trial_ci_pipeline.add_node(
     task=CreateTrialDatasetInventory, upstreams=CIWriteL1Frame, pip_extras=["inventory"]
 )
@@ -88,6 +96,8 @@ full_trial_ci_pipeline.add_node(task=CreateTrialAsdf, upstreams=CIWriteL1Frame,
 full_trial_ci_pipeline.add_node(
     task=CreateTrialQualityReport, upstreams=CIAssembleQualityData, pip_extras=["quality"]
 )
+
+# Output flow
 full_trial_ci_pipeline.add_node(
     task=TransferCryoTrialData,
     upstreams=[
@@ -97,8 +107,12 @@ full_trial_ci_pipeline.add_node(
         AssembleCryonirspMovie,
     ],
 )
+
+# goodbye
 full_trial_ci_pipeline.add_node(task=TrialTeardown, upstreams=TransferCryoTrialData)
 
+#######################
+#######################
 full_trial_sp_pipeline = Workflow(
     category="cryonirsp",
     input_data="l0",
@@ -107,6 +121,8 @@ full_trial_sp_pipeline = Workflow(
     workflow_package=__package__,
 )
 full_trial_sp_pipeline.add_node(task=TransferL0Data, upstreams=None)
+
+# Science flow
 full_trial_sp_pipeline.add_node(task=ParseL0CryonirspRampData, upstreams=TransferL0Data)
 full_trial_sp_pipeline.add_node(
     task=LinearityCorrection,
@@ -116,9 +132,6 @@ full_trial_sp_pipeline.add_node(
 full_trial_sp_pipeline.add_node(
     task=ParseL0CryonirspSPLinearizedData, upstreams=LinearityCorrection
 )
-full_trial_sp_pipeline.add_node(
-    task=CryonirspL0QualityMetrics, upstreams=ParseL0CryonirspSPLinearizedData
-)
 full_trial_sp_pipeline.add_node(
     task=BadPixelMapCalibration, upstreams=ParseL0CryonirspSPLinearizedData
 )
@@ -137,14 +150,23 @@ full_trial_sp_pipeline.add_node(
     upstreams=[SPInstrumentPolarizationCalibration, SPDispersionAxisCorrection],
 )
 full_trial_sp_pipeline.add_node(task=SPWriteL1Frame, upstreams=SPScienceCalibration)
-full_trial_sp_pipeline.add_node(task=QualityL1Metrics, upstreams=SPWriteL1Frame)
-full_trial_sp_pipeline.add_node(task=CryonirspL1QualityMetrics, upstreams=SPWriteL1Frame)
+
+# Movie flow
+full_trial_sp_pipeline.add_node(task=MakeCryonirspMovieFrames, upstreams=SPScienceCalibration)
+full_trial_sp_pipeline.add_node(task=AssembleCryonirspMovie, upstreams=MakeCryonirspMovieFrames)
+
+# Quality flow
+full_trial_sp_pipeline.add_node(
+    task=CryonirspL0QualityMetrics, upstreams=ParseL0CryonirspSPLinearizedData
+)
+full_trial_sp_pipeline.add_node(task=QualityL1Metrics, upstreams=SPScienceCalibration)
+full_trial_sp_pipeline.add_node(task=CryonirspL1QualityMetrics, upstreams=SPScienceCalibration)
 full_trial_sp_pipeline.add_node(
     task=SPAssembleQualityData,
     upstreams=[CryonirspL0QualityMetrics, QualityL1Metrics, CryonirspL1QualityMetrics],
 )
-full_trial_sp_pipeline.add_node(task=MakeCryonirspMovieFrames, upstreams=SPWriteL1Frame)
-full_trial_sp_pipeline.add_node(task=AssembleCryonirspMovie, upstreams=MakeCryonirspMovieFrames)
+
+# Trial data generation
 full_trial_sp_pipeline.add_node(
     task=CreateTrialDatasetInventory, upstreams=SPWriteL1Frame, pip_extras=["inventory"]
 )
@@ -152,6 +174,8 @@ full_trial_sp_pipeline.add_node(task=CreateTrialAsdf, upstreams=SPWriteL1Frame,
 full_trial_sp_pipeline.add_node(
     task=CreateTrialQualityReport, upstreams=SPAssembleQualityData, pip_extras=["quality"]
 )
+
+# Output flow
 full_trial_sp_pipeline.add_node(
     task=TransferCryoTrialData,
     upstreams=[
@@ -161,4 +185,6 @@ full_trial_sp_pipeline.add_node(
         AssembleCryonirspMovie,
     ],
 )
+
+# goodbye
 full_trial_sp_pipeline.add_node(task=TrialTeardown, upstreams=TransferCryoTrialData)

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/dkist_processing_cryonirsp.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-cryonirsp
-Version: 1.1.1rc1
+Version: 1.2.0
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/dkist_processing_cryonirsp.egg-info/SOURCES.txt

@@ -12,8 +12,6 @@ setup.cfg
 setup.py
 towncrier_science.toml
 changelog/.gitempty
-changelog/147.bugfix.rst
-changelog/148.bugfix.rst
 dkist_processing_cryonirsp/__init__.py
 dkist_processing_cryonirsp/config.py
 dkist_processing_cryonirsp.egg-info/PKG-INFO

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/dkist_processing_cryonirsp.egg-info/requires.txt

@@ -2,7 +2,7 @@ Pillow==10.3.0
 astropy==6.1.0
 dkist-fits-specifications==4.6.0
 dkist-header-validator==5.1.1
-dkist-processing-common==9.1.0
+dkist-processing-common==10.0.0
 dkist-processing-math==2.1.0
 dkist-processing-pac==3.0.2
 dkist-spectral-lines==3.0.0
@@ -39,7 +39,7 @@ dkist-processing-common[inventory]
 dkist-inventory==1.4.0
 
 [quality]
-dkist-quality==1.0.4
+dkist-quality==1.1.1
 
 [test]
 pytest
@@ -53,4 +53,4 @@ dkist-processing-common[inventory]
 dkist-inventory==1.4.0
 dkist-processing-common[asdf]
 dkist-inventory[asdf]==1.4.0
-dkist-quality==1.0.4
+dkist-quality==1.1.1

{dkist_processing_cryonirsp-1.1.1rc1 → dkist_processing_cryonirsp-1.2.0}/setup.cfg

@@ -21,7 +21,7 @@ install_requires =
 	astropy == 6.1.0
 	dkist-fits-specifications == 4.6.0
 	dkist-header-validator == 5.1.1
-	dkist-processing-common == 9.1.0
+	dkist-processing-common == 10.0.0
 	dkist-processing-math == 2.1.0
 	dkist-processing-pac == 3.0.2
 	dkist-spectral-lines == 3.0.0
@@ -54,7 +54,7 @@ asdf =
 	dkist-processing-common[asdf]
 	dkist-inventory[asdf]==1.4.0
 quality = 
-	dkist-quality==1.0.4
+	dkist-quality==1.1.1
 grogu = 
 	dkist
 	pyparsing

dkist_processing_cryonirsp-1.1.1rc1/changelog/147.bugfix.rst

@@ -1,2 +0,0 @@
-Add validation in linearity_correction task to ensure that the value of NUM_FRAMES_IN_RAMP is the same
-across all frames in a ramp, and that the value of NUM_FRAMES_IN_RAMP actually matches the number of frames found.

dkist_processing_cryonirsp-1.1.1rc1/changelog/148.bugfix.rst

@@ -1 +0,0 @@
-When writing linearized frames, use a fixed tag list as opposed to one derived from L0 data to alleviate load on redis.