dkist-processing-cryonirsp 0.0.59__tar.gz → 0.0.60__tar.gz

{dkist_processing_cryonirsp-0.0.59 → dkist_processing_cryonirsp-0.0.60}/CHANGELOG.rst

@@ -1,3 +1,12 @@
+v0.0.60 (2024-05-07)
+====================
+
+Features
+--------
+
+- Add the ability to create a quality report from a trial workflow. (`#121 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/121>`__)
+
+
 v0.0.59 (2024-05-06)
 ====================
 

{dkist_processing_cryonirsp-0.0.59 → dkist_processing_cryonirsp-0.0.60}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist_processing_cryonirsp
-Version: 0.0.59
+Version: 0.0.60
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA
@@ -13,6 +13,7 @@ Requires-Python: >=3.11
 Provides-Extra: test
 Provides-Extra: inventory
 Provides-Extra: asdf
+Provides-Extra: quality
 Provides-Extra: grogu
 Provides-Extra: docs
 

{dkist_processing_cryonirsp-0.0.59 → dkist_processing_cryonirsp-0.0.60}/dkist_processing_cryonirsp/tasks/l1_output_data.py

@@ -1,10 +1,10 @@
-"""Subclasses of SubmitQuality that cause the correct polcal metrics to build."""
-from dkist_processing_common.tasks import SubmitQuality
+"""Subclasses of AssembleQualityData that cause the correct polcal metrics to build."""
+from dkist_processing_common.tasks import AssembleQualityData
 
-__all__ = ["CISubmitQuality", "SPSubmitQuality"]
+__all__ = ["CIAssembleQualityData", "SPAssembleQualityData"]
 
 
-class CISubmitQuality(SubmitQuality):
+class CIAssembleQualityData(AssembleQualityData):
     """Subclass just so that the polcal_label_list can be populated."""
 
     @property
@@ -13,7 +13,7 @@ class CISubmitQuality(SubmitQuality):
         return ["CI Beam 1"]
 
 
-class SPSubmitQuality(SubmitQuality):
+class SPAssembleQualityData(AssembleQualityData):
     """Subclass just so that the polcal_label_list can be populated."""
 
     @property

{dkist_processing_cryonirsp-0.0.59 → dkist_processing_cryonirsp-0.0.60}/dkist_processing_cryonirsp/tasks/trial_output_data.py

@@ -52,4 +52,10 @@ class TransferCryoTrialData(TransferTrialDataBase):
         if self.output_asdf_switch:
             transfer_list += self.build_output_asdf_transfer_list()
 
+        if self.output_quality_report_switch:
+            transfer_list += self.build_output_quality_report_transfer_list()
+
+        if self.output_quality_data_switch:
+            transfer_list += self.build_output_quality_data_transfer_list()
+
         return transfer_list

{dkist_processing_cryonirsp-0.0.59 → dkist_processing_cryonirsp-0.0.60}/dkist_processing_cryonirsp/tests/local_trial_workflows/l0_cals_only.py

@@ -89,21 +89,20 @@ try:
     from dkist_inventory.asdf_generator import dataset_from_fits
 
     INV = True
-# Temp hack to work around import error caused by Stuart code updates...
-except (ModuleNotFoundError, ImportError):
-    # Bitbucket pipelines won't have dkist-inventory installed
-    pass
+except ModuleNotFoundError:
+    logger.warning(
+        "Could not load dkist-inventory. CreateTrialDatasetInventory and CreateTrialAsdf require dkist-inventory."
+    )
 
-QRM = False
+QUALITY = False
 try:
-    from quality_report_maker.libraries import report
-    from quality_report_maker.libraries.json_encoder import datetime_json_object_hook
-    from quality_report_maker.libraries.json_encoder import DatetimeEncoder
+    import dkist_quality
 
-    QRM = True
+    QUALITY = True
 except ModuleNotFoundError:
-    logger.warning("Could not find quality_report_maker (must be installed manually)")
-if QRM:
+    logger.warning("Could not load dkist-quality. CreateTrialQualityReport requires dkist-quality.")
+
+if QUALITY:
     import matplotlib.pyplot as plt
 
     plt.ioff()

{dkist_processing_cryonirsp-0.0.59 → dkist_processing_cryonirsp-0.0.60}/dkist_processing_cryonirsp/tests/local_trial_workflows/l0_to_l1.py

@@ -7,6 +7,8 @@ from pathlib import Path
 from astropy.io import fits
 from dkist_header_validator import spec214_validator
 from dkist_processing_common.manual import ManualProcessing
+from dkist_processing_common.tasks import CreateTrialAsdf
+from dkist_processing_common.tasks import CreateTrialQualityReport
 from dkist_processing_common.tasks import QualityL1Metrics
 from dkist_processing_common.tasks import WorkflowTaskBase
 from dkist_service_configuration import logger
@@ -27,7 +29,8 @@ from dkist_processing_cryonirsp.tasks.instrument_polarization import (
 from dkist_processing_cryonirsp.tasks.instrument_polarization import (
     SPInstrumentPolarizationCalibration,
 )
-from dkist_processing_cryonirsp.tasks.l1_output_data import SPSubmitQuality
+from dkist_processing_cryonirsp.tasks.l1_output_data import CIAssembleQualityData
+from dkist_processing_cryonirsp.tasks.l1_output_data import SPAssembleQualityData
 from dkist_processing_cryonirsp.tasks.linearity_correction import LinearityCorrection
 from dkist_processing_cryonirsp.tasks.make_movie_frames import MakeCryonirspMovieFrames
 from dkist_processing_cryonirsp.tasks.parse import ParseL0CryonirspLinearizedData
@@ -101,21 +104,20 @@ try:
     from dkist_inventory.asdf_generator import dataset_from_fits
 
     INV = True
-# Temp hack to work around import error caused by Stuart code updates...
-except (ModuleNotFoundError, ImportError):
-    # Bitbucket pipelines won't have dkist-inventory installed
-    pass
+except ModuleNotFoundError:
+    logger.warning(
+        "Could not load dkist-inventory. CreateTrialDatasetInventory and CreateTrialAsdf require dkist-inventory."
+    )
 
-QRM = False
+QUALITY = False
 try:
-    from quality_report_maker.libraries import report
-    from quality_report_maker.libraries.json_encoder import datetime_json_object_hook
-    from quality_report_maker.libraries.json_encoder import DatetimeEncoder
+    import dkist_quality
 
-    QRM = True
+    QUALITY = True
 except ModuleNotFoundError:
-    logger.warning("Could not find quality_report_maker (must be installed manually)")
-if QRM:
+    logger.warning("Could not load dkist-quality. CreateTrialQualityReport requires dkist-quality.")
+
+if QUALITY:
     import matplotlib.pyplot as plt
 
     plt.ioff()
@@ -168,29 +170,6 @@ class ShowExposureConditions(CryonirspTaskBase):
         logger.info(f"{self.constants.observe_exposure_conditions_list = }")
 
 
-class SubmitAndExposeQuality(SPSubmitQuality):
-    """A direct copy paste of SumbitQuality with an additional step of writing the report to disk"""
-
-    def run(self):
-        # To make sure the subclassing worked
-        super().run()
-
-        # Now regenerate the report str and save it
-        if QRM:
-            logger.info("Building quality report for save to disk")
-            report_str = self.quality_build_report(polcal_label_list=self.polcal_label_list)
-
-            doc_path = self.scratch.workflow_base_path / "quality_report.json"
-            report_container = {
-                "datasetId": self.constants.dataset_id,
-                "qualityReport": json.dumps(report_str, cls=DatetimeEncoder),
-            }
-            json_str = json.dumps(report_container)
-            with open(doc_path, "w") as f:
-                f.write(json_str)
-            logger.info(f"Wrote report to {doc_path}")
-
-
 class ValidateL1Output(CryonirspTaskBase):
     def run(self) -> None:
         files = self.read(tags=[CryonirspTag.output(), CryonirspTag.frame()])
@@ -311,38 +290,6 @@ def SP_workflow(
     manual_processing_run.run_task(task=AssembleCryonirspMovie)
 
 
-def make_pdf_report(scratch_path: str, recipe_run_id: int) -> None:
-    if not QRM:
-        logger.info("Did NOT make quality report pdf because quality_report_maker is not installed")
-        return
-
-    json_file = os.path.join(scratch_path, str(recipe_run_id), "quality_report.json")
-    pdf_file = os.path.join(scratch_path, str(recipe_run_id), "quality_report.pdf")
-    with open(json_file, "r") as f:
-        report_container = json.load(f)
-        dataset_id = report_container["datasetId"]
-        report_str = json.loads(
-            report_container["qualityReport"], object_hook=datetime_json_object_hook
-        )
-
-    pdf_bytes = report.format_report(report_str, f"GROGU_TEST_{dataset_id}")
-    with open(pdf_file, "wb") as f:
-        f.write(pdf_bytes)
-
-    logger.info(f"Wrote quality report PDF to {pdf_file}")
-
-
-def make_dataset_asdf(recipe_run_id, scratch_path):
-    if not INV:
-        logger.warning("Did NOT make dataset asdf file because dkist_inventory is not installed")
-        return
-
-    output_dir = os.path.join(scratch_path, str(recipe_run_id))
-    asdf_name = f"dataset_{recipe_run_id:03n}.asdf"
-    logger.info(f"Creating ASDF file from {output_dir} and saving to {asdf_name}")
-    dataset_from_fits(output_dir, asdf_name, hdu=1)
-
-
 def main(
     scratch_path: str,
     suffix: str = "FITS",
@@ -461,7 +408,10 @@ def main(
             raise ValueError(f"Did not recognize {arm_id = }")
         manual_processing_run.run_task(task=QualityL1Metrics)
         manual_processing_run.run_task(task=CryonirspL1QualityMetrics)
-        manual_processing_run.run_task(task=SubmitAndExposeQuality)
+        if arm_id == "SP":
+            manual_processing_run.run_task(task=SPAssembleQualityData)
+        elif arm_id == "CI":
+            manual_processing_run.run_task(task=CIAssembleQualityData)
         manual_processing_run.run_task(task=ValidateL1Output)
 
         if transfer_trial_data:
@@ -477,8 +427,17 @@ def main(
             manual_processing_run.run_task(transfer_local_task)
 
         # Test some downstream services
-        make_dataset_asdf(recipe_run_id, scratch_path)
-        make_pdf_report(scratch_path, recipe_run_id)
+        if INV:
+            manual_processing_run.run_task(task=CreateTrialAsdf)
+        else:
+            logger.warning(
+                "Did NOT make dataset asdf file because the asdf generator is not installed"
+            )
+
+        if QUALITY:
+            manual_processing_run.run_task(task=CreateTrialQualityReport)
+        else:
+            logger.warning("Did NOT make quality report pdf because dkist-quality is not installed")
 
         if any([load_dark, load_lamp, load_geometric, load_solar, load_inst_pol]):
             logger.info("NOT counting provenance records because some tasks were skipped")