dkist-processing-cryonirsp 0.0.65__tar.gz → 0.0.67__tar.gz

{dkist_processing_cryonirsp-0.0.65 → dkist_processing_cryonirsp-0.0.67}/CHANGELOG.rst

@@ -1,3 +1,23 @@
+v0.0.67 (2024-06-11)
+====================
+
+Misc
+----
+
+- Refactor production workflows to correct dependency of the `SubmitDatasetMetadata` task. (`#132 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/132>`__)
+- Remove trial workflows that don't create science data. (`#132 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/132>`__)
+
+
+v0.0.66 (2024-06-11)
+====================
+
+Misc
+----
+
+- Cast linearization threshold values to float 32 if they are found to be float64. (`#122 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/122>`__)
+- Add CRYO-NIRSP arm ID to the L1 filename. (`#131 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/131>`__)
+
+
 v0.0.65 (2024-06-04)
 ====================
 

{dkist_processing_cryonirsp-0.0.65 → dkist_processing_cryonirsp-0.0.67}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist_processing_cryonirsp
-Version: 0.0.65
+Version: 0.0.67
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA

{dkist_processing_cryonirsp-0.0.65 → dkist_processing_cryonirsp-0.0.67}/dkist_processing_cryonirsp/models/parameters.py

@@ -142,7 +142,11 @@ class CryonirspParameters(ParameterBase):
     def linearization_thresholds(self) -> np.ndarray:
         """Name of parameter associated with the linearization thresholds."""
         param_dict = self._find_parameter_for_arm("cryonirsp_linearization_thresholds")
-        return self._load_param_value_from_npy_file(param_dict)
+        value = self._load_param_value_from_npy_file(param_dict)
+        # float64 data can blow up the memory required for linearization - convert to float32
+        if np.issubdtype(value.dtype, np.float64):
+            value = value.astype(np.float32, casting="same_kind")
+        return value
 
     @cached_property
     def linearization_polyfit_coeffs(self) -> np.ndarray:

{dkist_processing_cryonirsp-0.0.65 → dkist_processing_cryonirsp-0.0.67}/dkist_processing_cryonirsp/tasks/write_l1.py

@@ -189,6 +189,34 @@ class CryonirspWriteL1Frame(WriteL1Frame, ABC):
                 f"Was supposed to be inserted during science calibration."
             ) from e
 
+    def l1_filename(self, header: fits.Header, stokes: Literal["I", "Q", "U", "V"]):
+        """
+        Use a FITS header to derive its filename in the following format.
+
+        instrument_arm_datetime_wavelength__stokes_datasetid_L1.fits.
+
+        This is done by taking the base filename and changing the instrument name to include the arm.
+
+        Example
+        -------
+        "CRYO-NIRSP_CI_2020_03_13T00_00_00_000_01080000_Q_DATID_L1.fits"
+
+        Parameters
+        ----------
+        header
+            The input fits header
+        stokes
+            The stokes parameter
+
+        Returns
+        -------
+        The L1 filename including the arm ID
+        """
+        base_l1_filename = super().l1_filename(header, stokes)
+        return base_l1_filename.replace(
+            self.constants.instrument, f"{self.constants.instrument}_{self.constants.arm_id}"
+        )
+
 
 class CIWriteL1Frame(CryonirspWriteL1Frame):
     """

{dkist_processing_cryonirsp-0.0.65 → dkist_processing_cryonirsp-0.0.67}/dkist_processing_cryonirsp/tests/conftest.py

@@ -93,6 +93,7 @@ def init_cryonirsp_constants_db():
         if is_dataclass(constants_obj):
             constants_obj = asdict(constants_obj)
         constants = CryonirspConstants(recipe_run_id=recipe_run_id, task_name="test")
+        constants._purge()
         constants._update(constants_obj)
         return
 
@@ -261,7 +262,7 @@ LINEARIZATION_THRESHOLDS_SP = "cryonirsp_linearization_thresholds_sp.npy"
 
 
 def _create_parameter_files(param_path: Path) -> None:
-    thresh = np.ones((10, 10), dtype=np.float32) * 100.0
+    thresh = np.ones((10, 10), dtype=np.float64) * 100.0
     np.save(os.path.join(param_path, LINEARIZATION_THRESHOLDS_CI), thresh)
     np.save(os.path.join(param_path, LINEARIZATION_THRESHOLDS_SP), thresh)
 

{dkist_processing_cryonirsp-0.0.65 → dkist_processing_cryonirsp-0.0.67}/dkist_processing_cryonirsp/tests/test_parameters.py

@@ -184,3 +184,20 @@ def test_other_parameters(basic_science_task_with_parameter_mixin):
         ):
             continue
         assert pv == getattr(task_params, pn.removeprefix("cryonirsp_"))
+
+
+@pytest.mark.parametrize("arm", [pytest.param("CI"), pytest.param("SP")])
+def test_linearization_threshold_parameters(
+    basic_science_task_with_parameter_mixin, arm, init_cryonirsp_constants_db
+):
+    """
+    Given: A Science task with the parameter mixin
+    When: Accessing properties for the linearization thresholds
+    Then: The correct type is returned
+    """
+    task, _ = basic_science_task_with_parameter_mixin
+    recipe_run_id = task.recipe_run_id
+    init_cryonirsp_constants_db(recipe_run_id, CryonirspConstantsDb(ARM_ID=arm))
+    linearization_threshold_array = task.parameters.linearization_thresholds
+
+    assert linearization_threshold_array.dtype == np.float32

{dkist_processing_cryonirsp-0.0.65 → dkist_processing_cryonirsp-0.0.67}/dkist_processing_cryonirsp/tests/test_write_l1.py

@@ -168,6 +168,7 @@ def test_write_l1_frame(write_l1_task, mocker, arm_id, num_stokes_params, num_ma
                     assert len(output_files) == 1
                     output_file = output_files.pop()
                     assert output_file.exists
+                    assert output_file.name.startswith(f"CRYO-NIRSP_{arm_id}_")
                     assert spec214_validator.validate(output_file, extra=False)
                     hdu_list = fits.open(output_file)
                     header = hdu_list[1].header

{dkist_processing_cryonirsp-0.0.65 → dkist_processing_cryonirsp-0.0.67}/dkist_processing_cryonirsp/workflows/ci_l0_processing.py

@@ -62,9 +62,8 @@ l0_pipeline.add_node(task=AssembleCryonirspMovie, upstreams=MakeCryonirspMovieFr
 l0_pipeline.add_node(
     task=SubmitDatasetMetadata, upstreams=[AssembleCryonirspMovie, CIAssembleQualityData]
 )
-l0_pipeline.add_node(task=TransferL1Data, upstreams=SubmitDatasetMetadata)
+l0_pipeline.add_node(task=TransferL1Data, upstreams=AssembleCryonirspMovie)
 l0_pipeline.add_node(
-    task=PublishCatalogAndQualityMessages,
-    upstreams=TransferL1Data,
+    task=PublishCatalogAndQualityMessages, upstreams=[SubmitDatasetMetadata, TransferL1Data]
 )
 l0_pipeline.add_node(task=Teardown, upstreams=PublishCatalogAndQualityMessages)

{dkist_processing_cryonirsp-0.0.65 → dkist_processing_cryonirsp-0.0.67}/dkist_processing_cryonirsp/workflows/sp_l0_processing.py

@@ -64,9 +64,8 @@ l0_pipeline.add_node(task=AssembleCryonirspMovie, upstreams=MakeCryonirspMovieFr
 l0_pipeline.add_node(
     task=SubmitDatasetMetadata, upstreams=[AssembleCryonirspMovie, SPAssembleQualityData]
 )
-l0_pipeline.add_node(task=TransferL1Data, upstreams=SubmitDatasetMetadata)
+l0_pipeline.add_node(task=TransferL1Data, upstreams=AssembleCryonirspMovie)
 l0_pipeline.add_node(
-    task=PublishCatalogAndQualityMessages,
-    upstreams=TransferL1Data,
+    task=PublishCatalogAndQualityMessages, upstreams=[SubmitDatasetMetadata, TransferL1Data]
 )
 l0_pipeline.add_node(task=Teardown, upstreams=PublishCatalogAndQualityMessages)

{dkist_processing_cryonirsp-0.0.65 → dkist_processing_cryonirsp-0.0.67}/dkist_processing_cryonirsp/workflows/trial_workflows.py

@@ -95,47 +95,6 @@ full_trial_ci_pipeline.add_node(
 )
 full_trial_ci_pipeline.add_node(task=TrialTeardown, upstreams=TransferCryoTrialData)
 
-non_science_ci_pipeline = Workflow(
-    category="cryonirsp",
-    input_data="l0",
-    output_data="l1",
-    detail="ci-non-science-trial",
-    workflow_package=__package__,
-)
-non_science_ci_pipeline.add_node(task=TransferL0Data, upstreams=None)
-non_science_ci_pipeline.add_node(task=ParseL0CryonirspRampData, upstreams=TransferL0Data)
-non_science_ci_pipeline.add_node(
-    task=LinearityCorrection,
-    resource_queue=ResourceQueue.HIGH_MEMORY,
-    upstreams=ParseL0CryonirspRampData,
-)
-non_science_ci_pipeline.add_node(task=ParseL0CryonirspLinearizedData, upstreams=LinearityCorrection)
-non_science_ci_pipeline.add_node(
-    task=CryonirspL0QualityMetrics, upstreams=ParseL0CryonirspLinearizedData
-)
-non_science_ci_pipeline.add_node(
-    task=BadPixelMapCalibration, upstreams=ParseL0CryonirspLinearizedData
-)
-non_science_ci_pipeline.add_node(task=CIBeamBoundariesCalibration, upstreams=BadPixelMapCalibration)
-non_science_ci_pipeline.add_node(task=DarkCalibration, upstreams=CIBeamBoundariesCalibration)
-non_science_ci_pipeline.add_node(task=LampGainCalibration, upstreams=DarkCalibration)
-non_science_ci_pipeline.add_node(task=CISolarGainCalibration, upstreams=LampGainCalibration)
-non_science_ci_pipeline.add_node(
-    task=CIInstrumentPolarizationCalibration, upstreams=CISolarGainCalibration
-)
-non_science_ci_pipeline.add_node(
-    task=CIAssembleQualityData,
-    upstreams=[CryonirspL0QualityMetrics, CIInstrumentPolarizationCalibration],
-)
-non_science_ci_pipeline.add_node(
-    task=CreateTrialQualityReport, upstreams=CIAssembleQualityData, pip_extras=["quality"]
-)
-non_science_ci_pipeline.add_node(
-    task=TransferCryoTrialData,
-    upstreams=CreateTrialQualityReport,
-)
-non_science_ci_pipeline.add_node(task=TrialTeardown, upstreams=TransferCryoTrialData)
-
 full_trial_sp_pipeline = Workflow(
     category="cryonirsp",
     input_data="l0",
@@ -194,45 +153,3 @@ full_trial_sp_pipeline.add_node(
     ],
 )
 full_trial_sp_pipeline.add_node(task=TrialTeardown, upstreams=TransferCryoTrialData)
-
-non_science_sp_pipeline = Workflow(
-    category="cryonirsp",
-    input_data="l0",
-    output_data="l1",
-    detail="sp-non-science-trial",
-    workflow_package=__package__,
-)
-non_science_sp_pipeline.add_node(task=TransferL0Data, upstreams=None)
-non_science_sp_pipeline.add_node(task=ParseL0CryonirspRampData, upstreams=TransferL0Data)
-non_science_sp_pipeline.add_node(
-    task=LinearityCorrection,
-    resource_queue=ResourceQueue.HIGH_MEMORY,
-    upstreams=ParseL0CryonirspRampData,
-)
-non_science_sp_pipeline.add_node(task=ParseL0CryonirspLinearizedData, upstreams=LinearityCorrection)
-non_science_sp_pipeline.add_node(
-    task=CryonirspL0QualityMetrics, upstreams=ParseL0CryonirspLinearizedData
-)
-non_science_sp_pipeline.add_node(
-    task=BadPixelMapCalibration, upstreams=ParseL0CryonirspLinearizedData
-)
-non_science_sp_pipeline.add_node(task=SPBeamBoundariesCalibration, upstreams=BadPixelMapCalibration)
-non_science_sp_pipeline.add_node(task=DarkCalibration, upstreams=SPBeamBoundariesCalibration)
-non_science_sp_pipeline.add_node(task=LampGainCalibration, upstreams=DarkCalibration)
-non_science_sp_pipeline.add_node(task=SPGeometricCalibration, upstreams=LampGainCalibration)
-non_science_sp_pipeline.add_node(task=SPSolarGainCalibration, upstreams=SPGeometricCalibration)
-non_science_sp_pipeline.add_node(
-    task=SPInstrumentPolarizationCalibration, upstreams=SPSolarGainCalibration
-)
-non_science_sp_pipeline.add_node(
-    task=SPAssembleQualityData,
-    upstreams=[CryonirspL0QualityMetrics, SPInstrumentPolarizationCalibration],
-)
-non_science_sp_pipeline.add_node(
-    task=CreateTrialQualityReport, upstreams=SPAssembleQualityData, pip_extras=["quality"]
-)
-non_science_sp_pipeline.add_node(
-    task=TransferCryoTrialData,
-    upstreams=CreateTrialQualityReport,
-)
-non_science_sp_pipeline.add_node(task=TrialTeardown, upstreams=TransferCryoTrialData)

{dkist_processing_cryonirsp-0.0.65 → dkist_processing_cryonirsp-0.0.67}/dkist_processing_cryonirsp.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-cryonirsp
-Version: 0.0.65
+Version: 0.0.67
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA