dkist-processing-test 1.9.14__tar.gz → 1.10.0rc1__tar.gz

{dkist-processing-test-1.9.14/dkist_processing_test.egg-info → dkist-processing-test-1.10.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-test
-Version: 1.9.14
+Version: 1.10.0rc1
 Summary: Example Instrument code that is used by the DKIST Science Data Processing pipelines to test processing infrastructure.
 Home-page: https://bitbucket.org/dkistdc/dkist-processing-test/src/main/
 Author: NSO / AURA
@@ -13,6 +13,7 @@ Requires-Python: >=3.11
 Provides-Extra: test
 Provides-Extra: inventory
 Provides-Extra: asdf
+Provides-Extra: quality
 
 dkist-processing-test
 ---------------------

{dkist-processing-test-1.9.14 → dkist-processing-test-1.10.0rc1}/dkist_processing_test/tasks/quality.py

@@ -1,9 +1,9 @@
 """Quality task definition."""
 from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.tasks import AssembleQualityData
 from dkist_processing_common.tasks import QualityL0Metrics
-from dkist_processing_common.tasks import SubmitQuality
 
-__all__ = ["TestQualityL0Metrics", "TestSubmitQuality"]
+__all__ = ["TestQualityL0Metrics", "TestAssembleQualityData"]
 
 
 class TestQualityL0Metrics(QualityL0Metrics):
@@ -12,7 +12,7 @@ class TestQualityL0Metrics(QualityL0Metrics):
         self.calculate_l0_metrics(paths=paths)
 
 
-class TestSubmitQuality(SubmitQuality):
+class TestAssembleQualityData(AssembleQualityData):
     @property
     def polcal_label_list(self) -> list[str] | None:
-        return ["Beam1"]
+        return ["Beam 1"]

{dkist-processing-test-1.9.14 → dkist-processing-test-1.10.0rc1}/dkist_processing_test/tasks/trial_output_data.py

@@ -33,6 +33,10 @@ class TransferTestTrialData(TransferTrialDataBase):
 
         transfer_list += self.build_output_asdf_transfer_list()
 
+        transfer_list += self.build_output_quality_data_transfer_list()
+
+        transfer_list += self.build_output_quality_report_transfer_list()
+
         transfer_list += self.build_output_movie_transfer_list()
 
         transfer_list += self.build_transfer_list_from_tag_lists([["FOO", "BAR"], ["BAZ"]])

{dkist-processing-test-1.9.14 → dkist-processing-test-1.10.0rc1}/dkist_processing_test/tests/test_tasks.py

@@ -488,6 +488,31 @@ def trial_output_task(recipe_run_id, tmp_path, mocker):
         task.write(output_file_obj, tags=[Tag.output(), Tag.frame()])
         file_count += 1
 
+        # Output dataset inventory
+        dsi_file_obj = uuid4().hex.encode("utf8")
+        task.write(dsi_file_obj, tags=[Tag.output(), Tag.dataset_inventory()])
+        file_count += 1
+
+        # Output asdf
+        asdf_file_obj = uuid4().hex.encode("utf8")
+        task.write(asdf_file_obj, tags=[Tag.output(), Tag.asdf()])
+        file_count += 1
+
+        # Output movie
+        movie_file_obj = uuid4().hex.encode("utf8")
+        task.write(movie_file_obj, tags=[Tag.output(), Tag.movie()])
+        file_count += 1
+
+        # Output quality data
+        quality_data_file_obj = uuid4().hex.encode("utf8")
+        task.write(quality_data_file_obj, tags=Tag.quality_data())
+        file_count += 1
+
+        # Output quality report
+        quality_report_file_obj = uuid4().hex.encode("utf8")
+        task.write(quality_report_file_obj, tags=[Tag.output(), Tag.quality_report()])
+        file_count += 1
+
         # Specifically tagged files
         task.write(uuid4().hex.encode("utf8"), tags=[Tag.frame(), "FOO", "BAR"])
         file_count += 1

{dkist-processing-test-1.9.14 → dkist-processing-test-1.10.0rc1}/dkist_processing_test/workflows/common_tasks.py

@@ -1,11 +1,12 @@
 """
 Workflow which exercises the common tasks in an end to end scenario
 """
-from dkist_processing_common.tasks import AddDatasetReceiptAccount
 from dkist_processing_common.tasks import CreateTrialAsdf
 from dkist_processing_common.tasks import CreateTrialDatasetInventory
+from dkist_processing_common.tasks import CreateTrialQualityReport
 from dkist_processing_common.tasks import PublishCatalogAndQualityMessages
 from dkist_processing_common.tasks import QualityL1Metrics
+from dkist_processing_common.tasks import SubmitDatasetMetadata
 from dkist_processing_common.tasks import Teardown
 from dkist_processing_common.tasks import TransferL0Data
 from dkist_processing_common.tasks import TransferL1Data
@@ -16,8 +17,8 @@ from dkist_processing_test.tasks import AssembleTestMovie
 from dkist_processing_test.tasks import GenerateCalibratedData
 from dkist_processing_test.tasks import MakeTestMovieFrames
 from dkist_processing_test.tasks import ParseL0TestInputData
+from dkist_processing_test.tasks import TestAssembleQualityData
 from dkist_processing_test.tasks import TestQualityL0Metrics
-from dkist_processing_test.tasks import TestSubmitQuality
 from dkist_processing_test.tasks import TransferTestTrialData
 from dkist_processing_test.tasks import WriteL1Data
 from dkist_processing_test.tasks.manual import ManualWithoutProvenance
@@ -63,15 +64,15 @@ quality_l1_metrics = Workflow(
 )
 quality_l1_metrics.add_node(task=QualityL1Metrics, upstreams=None)
 
-# SubmitQuality Task
-quality_submit_metrics = Workflow(
+# TestAssembleQualityData Task
+quality_assemble_data = Workflow(
     input_data="input",
     output_data="output",
     category="test",
-    detail="quality-submit-metrics",
+    detail="quality-assemble-data",
     workflow_package=__package__,
 )
-quality_submit_metrics.add_node(task=TestSubmitQuality, upstreams=None)
+quality_assemble_data.add_node(task=TestAssembleQualityData, upstreams=None)
 
 # GenerateL1CalibratedData Task
 generate_calibrated_data = Workflow(
@@ -133,15 +134,15 @@ transfer_trial_data = Workflow(
 )
 transfer_trial_data.add_node(task=TransferTestTrialData, upstreams=None)
 
-# AddDatasetReceiptAccount Task
-add_dataset_receipt_account = Workflow(
+# SubmitDatasetMetadata Task
+submit_dataset_metadata = Workflow(
     input_data="input",
     output_data="output",
     category="test",
-    detail="add-dataset-receipt-account",
+    detail="submit-dataset-metadata",
     workflow_package=__package__,
 )
-add_dataset_receipt_account.add_node(task=AddDatasetReceiptAccount, upstreams=None)
+submit_dataset_metadata.add_node(task=SubmitDatasetMetadata, upstreams=None)
 
 # PublishCatalogMessages Task
 publish_catalog_messages = Workflow(
@@ -195,6 +196,18 @@ create_trial_asdf = Workflow(
 )
 create_trial_asdf.add_node(task=CreateTrialAsdf, upstreams=None, pip_extras=["asdf"])
 
+# CreateTrialQualityReport Task
+create_trial_quality_report = Workflow(
+    input_data="input",
+    output_data="output",
+    category="test",
+    detail="create-trial-quality",
+    workflow_package=__package__,
+)
+create_trial_quality_report.add_node(
+    task=CreateTrialQualityReport, upstreams=None, pip_extras=["quality"]
+)
+
 
 # ManualWithProvenance Task
 manual_with_provenance = Workflow(

{dkist-processing-test-1.9.14 → dkist-processing-test-1.10.0rc1}/dkist_processing_test/workflows/end_to_end.py

@@ -1,10 +1,9 @@
 """
 Workflow which exercises the common tasks in an end to end scenario
 """
-from dkist_processing_common.tasks import AddDatasetReceiptAccount
 from dkist_processing_common.tasks import PublishCatalogAndQualityMessages
 from dkist_processing_common.tasks import QualityL1Metrics
-from dkist_processing_common.tasks import SubmitQuality
+from dkist_processing_common.tasks import SubmitDatasetMetadata
 from dkist_processing_common.tasks import Teardown
 from dkist_processing_common.tasks import TransferL0Data
 from dkist_processing_common.tasks import TransferL1Data
@@ -15,6 +14,7 @@ from dkist_processing_test.tasks import AssembleTestMovie
 from dkist_processing_test.tasks import GenerateCalibratedData
 from dkist_processing_test.tasks import MakeTestMovieFrames
 from dkist_processing_test.tasks import ParseL0TestInputData
+from dkist_processing_test.tasks import TestAssembleQualityData
 from dkist_processing_test.tasks import TestQualityL0Metrics
 from dkist_processing_test.tasks import WriteL1Data
 
@@ -35,10 +35,14 @@ end_to_end.add_node(
 )
 end_to_end.add_node(task=WriteL1Data, upstreams=GenerateCalibratedData)
 end_to_end.add_node(task=QualityL1Metrics, upstreams=WriteL1Data)
-end_to_end.add_node(task=SubmitQuality, upstreams=[QualityL1Metrics, TestQualityL0Metrics])
+end_to_end.add_node(
+    task=TestAssembleQualityData, upstreams=[TestQualityL0Metrics, QualityL1Metrics]
+)
 end_to_end.add_node(task=MakeTestMovieFrames, upstreams=WriteL1Data)
 end_to_end.add_node(task=AssembleTestMovie, upstreams=MakeTestMovieFrames)
-end_to_end.add_node(task=AddDatasetReceiptAccount, upstreams=[AssembleTestMovie, SubmitQuality])
-end_to_end.add_node(task=TransferL1Data, upstreams=AddDatasetReceiptAccount)
+end_to_end.add_node(
+    task=SubmitDatasetMetadata, upstreams=[AssembleTestMovie, TestAssembleQualityData]
+)
+end_to_end.add_node(task=TransferL1Data, upstreams=SubmitDatasetMetadata)
 end_to_end.add_node(task=PublishCatalogAndQualityMessages, upstreams=TransferL1Data)
 end_to_end.add_node(task=Teardown, upstreams=PublishCatalogAndQualityMessages)

{dkist-processing-test-1.9.14 → dkist-processing-test-1.10.0rc1}/dkist_processing_test/workflows/trial_end_to_end.py

@@ -3,6 +3,8 @@ Workflow which exercises the common tasks end to end in a trial scenario
 """
 from dkist_processing_common.tasks import CreateTrialAsdf
 from dkist_processing_common.tasks import CreateTrialDatasetInventory
+from dkist_processing_common.tasks import CreateTrialQualityReport
+from dkist_processing_common.tasks import QualityL1Metrics
 from dkist_processing_common.tasks import TransferL0Data
 from dkist_processing_common.tasks import TrialTeardown
 from dkist_processing_core import Workflow
@@ -11,6 +13,8 @@ from dkist_processing_test.tasks import AssembleTestMovie
 from dkist_processing_test.tasks import GenerateCalibratedData
 from dkist_processing_test.tasks import MakeTestMovieFrames
 from dkist_processing_test.tasks import ParseL0TestInputData
+from dkist_processing_test.tasks import TestAssembleQualityData
+from dkist_processing_test.tasks import TestQualityL0Metrics
 from dkist_processing_test.tasks import TransferTestTrialData
 from dkist_processing_test.tasks import WriteL1Data
 
@@ -21,16 +25,28 @@ trial = Workflow(
     detail="trial-e2e",
     workflow_package=__package__,
 )
+
 trial.add_node(task=TransferL0Data, upstreams=None)
 trial.add_node(task=ParseL0TestInputData, upstreams=TransferL0Data)
+trial.add_node(task=TestQualityL0Metrics, upstreams=ParseL0TestInputData)
 trial.add_node(task=GenerateCalibratedData, upstreams=ParseL0TestInputData)
 trial.add_node(task=WriteL1Data, upstreams=GenerateCalibratedData)
+trial.add_node(task=QualityL1Metrics, upstreams=WriteL1Data)
 trial.add_node(task=MakeTestMovieFrames, upstreams=WriteL1Data)
 trial.add_node(task=AssembleTestMovie, upstreams=MakeTestMovieFrames)
+trial.add_node(task=TestAssembleQualityData, upstreams=[TestQualityL0Metrics, QualityL1Metrics])
 trial.add_node(task=CreateTrialDatasetInventory, upstreams=WriteL1Data, pip_extras=["inventory"])
 trial.add_node(task=CreateTrialAsdf, upstreams=WriteL1Data, pip_extras=["asdf"])
+trial.add_node(
+    task=CreateTrialQualityReport, upstreams=TestAssembleQualityData, pip_extras=["quality"]
+)
 trial.add_node(
     task=TransferTestTrialData,
-    upstreams=[CreateTrialDatasetInventory, CreateTrialAsdf, AssembleTestMovie],
+    upstreams=[
+        CreateTrialDatasetInventory,
+        CreateTrialAsdf,
+        CreateTrialQualityReport,
+        AssembleTestMovie,
+    ],
 )
 trial.add_node(task=TrialTeardown, upstreams=TransferTestTrialData)

{dkist-processing-test-1.9.14 → dkist-processing-test-1.10.0rc1/dkist_processing_test.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-test
-Version: 1.9.14
+Version: 1.10.0rc1
 Summary: Example Instrument code that is used by the DKIST Science Data Processing pipelines to test processing infrastructure.
 Home-page: https://bitbucket.org/dkistdc/dkist-processing-test/src/main/
 Author: NSO / AURA
@@ -13,6 +13,7 @@ Requires-Python: >=3.11
 Provides-Extra: test
 Provides-Extra: inventory
 Provides-Extra: asdf
+Provides-Extra: quality
 
 dkist-processing-test
 ---------------------

{dkist-processing-test-1.9.14 → dkist-processing-test-1.10.0rc1}/dkist_processing_test.egg-info/requires.txt

@@ -1,4 +1,4 @@
-dkist-processing-common==6.2.1
+dkist-processing-common==6.2.0rc3
 dkist-header-validator==5.1.0
 dkist-service-configuration==1.1.0
 dkist-fits-specifications==4.1.1
@@ -15,6 +15,9 @@ dkist-inventory[asdf]==1.3.1
 dkist-processing-common[inventory]
 dkist-inventory==1.3.1
 
+[quality]
+dkist-quality==1.0.1
+
 [test]
 pytest
 pytest-xdist
@@ -25,3 +28,4 @@ dkist-processing-common[inventory]
 dkist-inventory==1.3.1
 dkist-processing-common[asdf]
 dkist-inventory[asdf]==1.3.1
+dkist-quality==1.0.1

{dkist-processing-test-1.9.14 → dkist-processing-test-1.10.0rc1}/setup.cfg

@@ -17,7 +17,7 @@ setup_requires = setuptools_scm
 packages = find:
 include_package_data = True
 install_requires = 
-	dkist-processing-common == 6.2.1
+	dkist-processing-common == 6.2.0rc3
 	dkist-header-validator == 5.1.0
 	dkist-service-configuration == 1.1.0
 	dkist-fits-specifications == 4.1.1
@@ -35,12 +35,15 @@ test =
 	dkist-data-simulator >= 5.1.1
 	%(inventory)s  # identify pip conflicts in test
 	%(asdf)s  # identify pip conflicts in test
+	%(quality)s  # identify pip conflicts in test
 inventory = 
 	dkist-processing-common[inventory]
 	dkist-inventory==1.3.1
 asdf = 
 	dkist-processing-common[asdf]
 	dkist-inventory[asdf]==1.3.1
+quality = 
+	dkist-quality == 1.0.1
 
 [tool:pytest]
 markers =