dkist-processing-cryonirsp 0.0.81__tar.gz → 1.0.0__tar.gz

{dkist_processing_cryonirsp-0.0.81 → dkist_processing_cryonirsp-1.0.0}/CHANGELOG.rst

@@ -1,3 +1,21 @@
+v1.0.0 (2024-08-21)
+===================
+
+Misc
+----
+
+- CRYO-NIRSP processing pipeline data accepted for release to the community.
+
+
+v0.0.82 (2024-08-21)
+====================
+
+Misc
+----
+
+- Update some Quality related tasks and methods for the new API in `dkist-processing-common` v9.0.0. No change to any outputs. (`#146 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/146>`__)
+
+
 v0.0.81 (2024-08-16)
 ====================
 

{dkist_processing_cryonirsp-0.0.81 → dkist_processing_cryonirsp-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist_processing_cryonirsp
-Version: 0.0.81
+Version: 1.0.0
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA

dkist_processing_cryonirsp-1.0.0/SCIENCE_CHANGELOG.rst

@@ -0,0 +1,16 @@
+v1.0.0 (2024-08-21)
+===================
+
+Misc
+----
+
+- CRYO-NIRSP processing pipeline data accepted for release to the community.
+
+
+v0.0.1 (2023-08-25)
+===================
+
+Misc
+----
+
+- Initial release of pipeline for science review

{dkist_processing_cryonirsp-0.0.81 → dkist_processing_cryonirsp-1.0.0}/dkist_processing_cryonirsp/tasks/instrument_polarization.py

@@ -412,10 +412,11 @@ class CIInstrumentPolarizationCalibration(InstrumentPolarizationCalibrationBase)
         self.quality_store_polcal_results(
             polcal_fitter=polcal_fitter,
             label=generate_polcal_quality_label(arm="CI", beam=beam),
-            bins_1=self.parameters.polcal_num_spatial_bins,
-            bins_2=self.parameters.polcal_num_spatial_bins,
-            bin_1_type="spatial",
-            bin_2_type="spatial",
+            bin_nums=[
+                self.parameters.polcal_num_spatial_bins,
+                self.parameters.polcal_num_spatial_bins,
+            ],
+            bin_labels=["spatial", "spatial"],
         )
 
 
@@ -431,8 +432,9 @@ class SPInstrumentPolarizationCalibration(InstrumentPolarizationCalibrationBase)
         self.quality_store_polcal_results(
             polcal_fitter=polcal_fitter,
             label=generate_polcal_quality_label(arm="SP", beam=beam),
-            bins_1=self.parameters.polcal_num_spatial_bins,
-            bins_2=self.parameters.polcal_num_spectral_bins,
-            bin_1_type="spatial",
-            bin_2_type="spectral",
+            bin_nums=[
+                self.parameters.polcal_num_spatial_bins,
+                self.parameters.polcal_num_spectral_bins,
+            ],
+            bin_labels=["spatial", "spectral"],
         )

{dkist_processing_cryonirsp-0.0.81 → dkist_processing_cryonirsp-1.0.0}/dkist_processing_cryonirsp/tasks/quality_metrics.py

@@ -2,6 +2,7 @@
 from dataclasses import dataclass
 from dataclasses import field
 from typing import Generator
+from typing import Iterable
 
 import numpy as np
 from astropy.time import Time
@@ -66,16 +67,26 @@ class CryonirspL0QualityMetrics(QualityL0Metrics):
         """Class for Cryonirsp constants."""
         return CryonirspConstants
 
-    def run(self) -> None:
-        """Calculate L0 metrics for Cryonirsp data."""
-        if not self.constants.correct_for_polarization:
-            paths = self.read(tags=[CryonirspTag.linearized()])
-            self.calculate_l0_metrics(paths=paths)
-        else:
-            for m in range(1, self.constants.num_modstates + 1):
-                with self.apm_task_step(f"Working on modstate {m}"):
-                    paths = self.read(tags=[CryonirspTag.linearized(), CryonirspTag.modstate(m)])
-                    self.calculate_l0_metrics(paths=paths, modstate=m)
+    @property
+    def raw_frame_tag(self) -> str:
+        """
+        Define tag corresponding to L0 data.
+
+        For Cryo it's LINEARIZED.
+        """
+        return CryonirspTag.linearized()
+
+    @property
+    def modstate_list(self) -> Iterable[int] | None:
+        """
+        Define the list of modstates over which to compute L0 quality metrics.
+
+        If the dataset is non-polarimetric then we just compute all metrics over all modstates at once.
+        """
+        if self.constants.correct_for_polarization:
+            return range(1, self.constants.num_modstates + 1)
+
+        return None
 
 
 class CryonirspL1QualityMetrics(CryonirspTaskBase, QualityMixin):

{dkist_processing_cryonirsp-0.0.81 → dkist_processing_cryonirsp-1.0.0}/dkist_processing_cryonirsp/tests/test_assemble_qualilty.py

@@ -230,8 +230,7 @@ def quality_metrics(dataframe_json) -> list[Metric]:
                     "I_sys_CS00_step01": {"fit_values": [10, 20, 30.0], "init_value": 0.33},
                     "param_X": {"fit_values": [5, 6, 7.0], "init_value": 99},
                 },
-                "bin_1_str": "bin1",
-                "bin_2_str": "bin2",
+                "bin_strs": ["bin1", "bin2"],
                 "total_bins": 100,
                 "num_varied_I_sys": 2,
             },
@@ -239,8 +238,7 @@ def quality_metrics(dataframe_json) -> list[Metric]:
         ),
         Metric(
             {
-                "bin_1_str": "bin1",
-                "bin_2_str": "bin2",
+                "bin_strs": ["bin1", "bin2"],
                 "total_bins": 100,
                 "red_chi_list": [1, 2, 3],
                 "residual_json": dataframe_json,
@@ -249,8 +247,7 @@ def quality_metrics(dataframe_json) -> list[Metric]:
         ),
         Metric(
             {
-                "bin_1_str": "bin1",
-                "bin_2_str": "bin2",
+                "bin_strs": ["bin1", "bin2"],
                 "total_bins": 100,
                 "efficiency_list": ((np.random.randn(4, 100) - 0.5) * 0.3).tolist(),
                 "warnings": ["A warning"],

{dkist_processing_cryonirsp-0.0.81 → dkist_processing_cryonirsp-1.0.0}/dkist_processing_cryonirsp/tests/test_instrument_polarization.py

@@ -183,7 +183,8 @@ def ci_instrument_polarization_calibration_task(
 
             # Don't test place-holder QA stuff for now
             quality_metric_mocker = mocker.patch(
-                "dkist_processing_cryonirsp.tasks.instrument_polarization.CIInstrumentPolarizationCalibration.quality_store_polcal_results"
+                "dkist_processing_cryonirsp.tasks.instrument_polarization.CIInstrumentPolarizationCalibration.quality_store_polcal_results",
+                autospec=True,
             )
 
             # Create beam border intermediate array that is consistent with a single pixel array
@@ -297,7 +298,8 @@ def sp_instrument_polarization_calibration_task(
 
             # Don't test place-holder QA stuff for now
             quality_metric_mocker = mocker.patch(
-                "dkist_processing_cryonirsp.tasks.instrument_polarization.SPInstrumentPolarizationCalibration.quality_store_polcal_results"
+                "dkist_processing_cryonirsp.tasks.instrument_polarization.SPInstrumentPolarizationCalibration.quality_store_polcal_results",
+                autospec=True,
             )
 
             # Create beam border intermediate arrays that are consistent with a single pixel array
@@ -410,12 +412,14 @@ def test_ci_instrument_polarization_calibration_task(
         assert len(list(task.read(tags=tags))) == 1
 
         quality_mocker.assert_any_call(
+            task,
             polcal_fitter=ANY,
             label=f"CI Beam {beam}",
-            bins_1=task.parameters.polcal_num_spatial_bins,
-            bins_2=task.parameters.polcal_num_spatial_bins,
-            bin_1_type="spatial",
-            bin_2_type="spatial",
+            bin_nums=[
+                task.parameters.polcal_num_spatial_bins,
+                task.parameters.polcal_num_spatial_bins,
+            ],
+            bin_labels=["spatial", "spatial"],
         )
 
 
@@ -446,13 +450,14 @@ def test_sp_instrument_polarization_calibration_task(
         assert len(list(task.read(tags=tags))) == 1
 
         quality_mocker.assert_any_call(
+            task,
             polcal_fitter=ANY,
             label=f"SP Beam {beam}",
-            bins_1=task.parameters.polcal_num_spatial_bins,
-            bins_2=task.parameters.polcal_num_spectral_bins,
-            bin_1_type="spatial",
-            bin_2_type="spectral",
-            # skip_recording_constant_pars=beam == 2,
+            bin_nums=[
+                task.parameters.polcal_num_spatial_bins,
+                task.parameters.polcal_num_spectral_bins,
+            ],
+            bin_labels=["spatial", "spectral"],
         )
 
 

{dkist_processing_cryonirsp-0.0.81 → dkist_processing_cryonirsp-1.0.0}/dkist_processing_cryonirsp/tests/test_quality.py

@@ -1,21 +1,87 @@
 import json
 
+import numpy as np
 import pytest
 from astropy.io import fits
 from dkist_header_validator import spec122_validator
 from dkist_processing_common._util.scratch import WorkflowFileSystem
+from dkist_processing_common.codecs.fits import fits_array_encoder
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
+from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_common.tests.conftest import FakeGQLClient
 
 from dkist_processing_cryonirsp.models.tags import CryonirspTag
+from dkist_processing_cryonirsp.tasks.quality_metrics import CryonirspL0QualityMetrics
 from dkist_processing_cryonirsp.tasks.quality_metrics import CryonirspL1QualityMetrics
 from dkist_processing_cryonirsp.tests.conftest import CryonirspConstantsDb
 from dkist_processing_cryonirsp.tests.conftest import generate_214_l1_fits_frame
 from dkist_processing_cryonirsp.tests.header_models import Cryonirsp122ObserveFrames
+from dkist_processing_cryonirsp.tests.header_models import CryonirspHeaders
+
+
+@pytest.fixture
+def cryonirsp_l0_quality_task(recipe_run_id, num_modstates, init_cryonirsp_constants_db, tmp_path):
+    constants = CryonirspConstantsDb(
+        NUM_MODSTATES=num_modstates,
+        MODULATOR_SPIN_MODE="Stepped" if num_modstates > 1 else "Off",
+    )
+    init_cryonirsp_constants_db(recipe_run_id, constants)
+
+    with CryonirspL0QualityMetrics(
+        recipe_run_id=recipe_run_id,
+        workflow_name="workflow_name",
+        workflow_version="workflow_version",
+    ) as task:
+        task.scratch = WorkflowFileSystem(scratch_base_path=tmp_path, recipe_run_id=recipe_run_id)
+
+        yield task
+        task._purge()
+
+
+@pytest.fixture()
+def l0_quality_task_types() -> list[str]:
+    # The tasks types we want to build l0 metrics for
+    return [TaskName.lamp_gain.value, TaskName.dark.value]
+
+
+@pytest.fixture()
+def dataset_task_types(l0_quality_task_types) -> list[str]:
+    # The task types that exist in the dataset. I.e., a larger set than we want to build metrics for.
+    return l0_quality_task_types + [TaskName.solar_gain.value, TaskName.observe.value]
+
+
+@pytest.fixture
+def write_l0_task_frames_to_task(num_modstates, dataset_task_types):
+    array_shape = (1, 4, 4)
+    data = np.ones(array_shape)
+
+    def writer(task):
+        for task_type in dataset_task_types:
+            ds = CryonirspHeaders(
+                dataset_shape=(num_modstates,) + array_shape[-2:],
+                array_shape=array_shape,
+                time_delta=1.0,
+                file_schema="level0_spec214",
+            )
+            for modstate, frame in enumerate(ds, start=1):
+                header = frame.header()
+                task.write(
+                    data=data,
+                    header=header,
+                    tags=[
+                        CryonirspTag.linearized(),
+                        CryonirspTag.frame(),
+                        CryonirspTag.task(task_type),
+                        CryonirspTag.modstate(modstate),
+                    ],
+                    encoder=fits_array_encoder,
+                )
+
+    return writer
 
 
 @pytest.fixture(scope="function")
-def cryo_quality_task(tmp_path, recipe_run_id, init_cryonirsp_constants_db):
+def cryo_l1_quality_task(tmp_path, recipe_run_id, init_cryonirsp_constants_db):
     num_map_scans = 3
     num_scan_steps = 1
     constants_db = CryonirspConstantsDb(
@@ -63,9 +129,42 @@ def cryo_quality_task(tmp_path, recipe_run_id, init_cryonirsp_constants_db):
         task._purge()
 
 
-def test_quality_task(cryo_quality_task, mocker):
+@pytest.mark.parametrize(
+    "num_modstates", [pytest.param(4, id="polarimetric"), pytest.param(1, id="intensity")]
+)
+def test_l0_quality_task(
+    cryonirsp_l0_quality_task, num_modstates, write_l0_task_frames_to_task, l0_quality_task_types
+):
+    """
+    Given: A `CryonirspL0QualityMetrics` task and some INPUT frames tagged with their task type and modstate
+    When: Running the task
+    Then: The expected L0 quality metric files exist
+    """
+    # NOTE: We rely on the unit tests in `*-common` to verify the correct format/data of the metric files
+    task = cryonirsp_l0_quality_task
+    write_l0_task_frames_to_task(task)
+
+    task()
+
+    task_metric_names = ["FRAME_RMS", "FRAME_AVERAGE"]
+    for metric_name in task_metric_names:
+        for modstate in range(1, num_modstates + 1):
+            for task_type in l0_quality_task_types:
+                tags = [CryonirspTag.quality(metric_name), CryonirspTag.quality_task(task_type)]
+                if num_modstates > 1:
+                    tags.append(CryonirspTag.modstate(modstate))
+                files = list(task.read(tags))
+                assert len(files) == 1
+
+    global_metric_names = ["DATASET_AVERAGE", "DATASET_RMS"]
+    for metric_name in global_metric_names:
+        files = list(task.read(tags=[CryonirspTag.quality(metric_name)]))
+        assert len(files) > 0
+
+
+def test_l1_quality_task(cryo_l1_quality_task, mocker):
     """
-    Given: A CryonirspQualityMetrics task
+    Given: A CryonirspL1QualityMetrics task
     When: Calling the task instance
     Then: A single sensitivity measurement and datetime is recorded for each map scan for each Stokes Q, U, and V,
             and a single noise measurement and datetime is recorded for L1 file for each Stokes Q, U, and V
@@ -74,7 +173,7 @@ def test_quality_task(cryo_quality_task, mocker):
         "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
     )
     # When
-    task = cryo_quality_task
+    task = cryo_l1_quality_task
     task()
     # Then
     num_map_scans = task.constants.num_map_scans

{dkist_processing_cryonirsp-0.0.81 → dkist_processing_cryonirsp-1.0.0}/dkist_processing_cryonirsp.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-cryonirsp
-Version: 0.0.81
+Version: 1.0.0
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA

{dkist_processing_cryonirsp-0.0.81 → dkist_processing_cryonirsp-1.0.0}/dkist_processing_cryonirsp.egg-info/requires.txt

@@ -2,7 +2,7 @@ Pillow==10.3.0
 astropy==6.1.0
 dkist-fits-specifications==4.6.0
 dkist-header-validator==5.1.1
-dkist-processing-common==8.2.2
+dkist-processing-common==9.0.0
 dkist-processing-math==2.1.0
 dkist-processing-pac==3.0.2
 dkist-spectral-lines==3.0.0

{dkist_processing_cryonirsp-0.0.81 → dkist_processing_cryonirsp-1.0.0}/setup.cfg

@@ -21,7 +21,7 @@ install_requires =
 	astropy == 6.1.0
 	dkist-fits-specifications == 4.6.0
 	dkist-header-validator == 5.1.1
-	dkist-processing-common == 8.2.2
+	dkist-processing-common == 9.0.0
 	dkist-processing-math == 2.1.0
 	dkist-processing-pac == 3.0.2
 	dkist-spectral-lines == 3.0.0

dkist_processing_cryonirsp-0.0.81/SCIENCE_CHANGELOG.rst

@@ -1,7 +0,0 @@
-v0.0.1 (2023-08-25)
-===================
-
-Misc
-----
-
-- Initial release of pipeline for science review