dkist-processing-common 12.2.0__py3-none-any.whl → 12.2.0rc1__py3-none-any.whl

changelog/272.feature.rst

@@ -0,0 +1 @@
+Add the framework for using Dataset Extras, that is, other data that can be included with the L1 FITS files generated by the regular pipeline. A new abstract class, WriteL1DatasetExtras, provides helper functionality for use in the instrument pipelines.

dkist_processing_common/models/constants.py

@@ -473,14 +473,14 @@ class ConstantsBase:
         return self._db_dict[BudName.solar_gain_gos_polarizer_status]
 
     @property
-    def dark_gos_polarizer_angle(self) -> str:
+    def dark_gos_polarizer_angle(self) -> float:
         """Return the gos polarizer angle constant for the dark task."""
-        return str(self._db_dict[BudName.dark_gos_polarizer_angle])
+        return self._db_dict[BudName.dark_gos_polarizer_angle]
 
     @property
-    def solar_gain_gos_polarizer_angle(self) -> str:
+    def solar_gain_gos_polarizer_angle(self) -> float:
         """Return the gos polarizer angle constant for the solar gain task."""
-        return str(self._db_dict[BudName.solar_gain_gos_polarizer_angle])
+        return self._db_dict[BudName.solar_gain_gos_polarizer_angle]
 
     @property
     def dark_gos_retarder_status(self) -> str:
@@ -493,14 +493,14 @@ class ConstantsBase:
         return self._db_dict[BudName.solar_gain_gos_retarder_status]
 
     @property
-    def dark_gos_retarder_angle(self) -> str:
+    def dark_gos_retarder_angle(self) -> float:
         """Return the gos retarder angle constant for the dark task."""
-        return str(self._db_dict[BudName.dark_gos_retarder_angle])
+        return self._db_dict[BudName.dark_gos_retarder_angle]
 
     @property
-    def solar_gain_gos_retarder_angle(self) -> str:
+    def solar_gain_gos_retarder_angle(self) -> float:
         """Return the gos retarder angle constant for the solar gain task."""
-        return str(self._db_dict[BudName.solar_gain_gos_retarder_angle])
+        return self._db_dict[BudName.solar_gain_gos_retarder_angle]
 
     @property
     def dark_gos_level0_status(self) -> str:

dkist_processing_common/tasks/l1_output_data.py

@@ -2,14 +2,18 @@
 
 import logging
 from abc import ABC
+from itertools import chain
 from pathlib import Path
 from typing import Iterable
 
+from dkist_processing_common.codecs.quality import quality_data_decoder
 from dkist_processing_common.codecs.quality import quality_data_encoder
 from dkist_processing_common.models.message import CatalogFrameMessage
 from dkist_processing_common.models.message import CatalogFrameMessageBody
 from dkist_processing_common.models.message import CatalogObjectMessage
 from dkist_processing_common.models.message import CatalogObjectMessageBody
+from dkist_processing_common.models.message import CreateQualityReportMessage
+from dkist_processing_common.models.message import CreateQualityReportMessageBody
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.mixin.globus import GlobusMixin
 from dkist_processing_common.tasks.mixin.interservice_bus import InterserviceBusMixin
@@ -189,9 +193,7 @@ class SubmitDatasetMetadata(L1OutputDataBase):
 class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
     """Task class for publishing Catalog and Quality Messages."""
 
-    def frame_messages(
-        self, paths: Iterable[Path], folder_modifier: str | None = None
-    ) -> list[CatalogFrameMessage]:
+    def frame_messages(self, paths: Iterable[Path]) -> list[CatalogFrameMessage]:
         """
         Create the frame messages.
 
@@ -199,8 +201,6 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         ----------
         paths
             The input paths for which to publish frame messages
-        folder_modifier
-            A subdirectory to use if the files in paths are not in the base directory
 
         Returns
         -------
@@ -208,7 +208,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         """
         message_bodies = [
             CatalogFrameMessageBody(
-                objectName=self.format_object_key(path=p, folder_modifier=folder_modifier),
+                objectName=self.format_object_key(path=p),
                 conversationId=str(self.recipe_run_id),
                 bucket=self.destination_bucket,
             )
@@ -237,7 +237,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         message_bodies = [
             CatalogObjectMessageBody(
                 objectType=object_type,
-                objectName=self.format_object_key(path=p),
+                objectName=self.format_object_key(p),
                 bucket=self.destination_bucket,
                 conversationId=str(self.recipe_run_id),
                 groupId=self.constants.dataset_id,
@@ -260,7 +260,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
             messages = []
             messages += self.frame_messages(paths=frames)
             frame_message_count = len(messages)
-            messages += self.frame_messages(paths=extras, folder_modifier="extra")
+            messages += self.frame_messages(paths=extras)
             extra_message_count = len(extras)
             messages += self.object_messages(paths=movies, object_type="MOVIE")
             object_message_count = len(movies)
@@ -270,4 +270,14 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         with self.telemetry_span(
             f"Publish messages: {frame_message_count = }, {extra_message_count = }, {object_message_count = }, {dataset_has_quality_data = }"
         ):
+            messages.append(self.quality_report_message)
+        with self.telemetry_span("Publish Catalog Messages") as publish_span:
+            publish_span.set_attributes(
+                {
+                    "frame_message_count": frame_message_count,
+                    "extra_message_count": extra_message_count,
+                    "object_message_count": object_message_count,
+                    "dataset_has_quality_data": dataset_has_quality_data,
+                }
+            )
             self.interservice_bus_publish(messages=messages)

dkist_processing_common/tasks/write_extra.py

@@ -27,7 +27,6 @@ class WriteL1DatasetExtras(WriteL1Base, ABC):
         filename: str,
         task_type: TaskName,
         extra_name: DatasetExtraType,
-        end_time: str,
         total_exposure: float | None = None,
         readout_exposure: float | None = None,
     ) -> dict:
@@ -39,6 +38,7 @@ class WriteL1DatasetExtras(WriteL1Base, ABC):
                     self.constants.dark_observing_program_execution_ids
                 )
                 task_specific_date_begin = self.constants.dark_date_begin
+                task_specific_date_end = self.constants.dark_date_end
                 task_specific_raw_frames_per_fpa = (
                     0  # can be updated in construction of dataset extra if required
                 )
@@ -74,6 +74,7 @@ class WriteL1DatasetExtras(WriteL1Base, ABC):
                 )
 
                 task_specific_date_begin = self.constants.solar_gain_date_begin
+                task_specific_date_end = self.constants.solar_gain_date_end
                 task_specific_raw_frames_per_fpa = self.constants.solar_gain_num_raw_frames_per_fpa
                 task_specific_telescope_tracking_mode = (
                     self.constants.solar_gain_telescope_tracking_mode
@@ -109,6 +110,7 @@ class WriteL1DatasetExtras(WriteL1Base, ABC):
                 )
 
                 task_specific_date_begin = self.constants.polcal_date_begin
+                task_specific_date_end = self.constants.polcal_date_end
                 task_specific_raw_frames_per_fpa = self.constants.polcal_num_raw_frames_per_fpa
                 task_specific_telescope_tracking_mode = (
                     self.constants.polcal_telescope_tracking_mode
@@ -140,14 +142,14 @@ class WriteL1DatasetExtras(WriteL1Base, ABC):
                 raise ValueError(f"Unsupported task type {task_type}")
 
         start_datetime = datetime.fromisoformat(task_specific_date_begin)
-        end_datetime = datetime.fromisoformat(end_time)
+        end_datetime = datetime.fromisoformat(task_specific_date_end)
 
         dataset_extra_header = {
             DatasetExtraHeaderSection.common: {
                 "BUNIT": "count",
                 "DATE": Time.now().fits,
                 "DATE-BEG": task_specific_date_begin,
-                "DATE-END": end_time,
+                "DATE-END": task_specific_date_end,
                 "TELAPSE": (end_datetime - start_datetime).total_seconds(),
                 "DATE-AVG": (start_datetime + (end_datetime - start_datetime) / 2).isoformat(),
                 "TIMESYS": "UTC",
@@ -263,7 +265,6 @@ class WriteL1DatasetExtras(WriteL1Base, ABC):
         extra_name: DatasetExtraType,
         total_exposure: float | None = None,
         readout_exposure: float | None = None,
-        end_time: str | None = None,
     ) -> fits.Header:
         """Build FITS header for dataset extra file."""
         header = fits.Header()
@@ -273,7 +274,6 @@ class WriteL1DatasetExtras(WriteL1Base, ABC):
             total_exposure=total_exposure,
             readout_exposure=readout_exposure,
             extra_name=extra_name,
-            end_time=end_time,
         )
         for section in sections:
             header.update(all_section_headers[section].items())

dkist_processing_common/tests/test_construct_dataset_extras.py

@@ -118,7 +118,6 @@ class ConstructDatasetExtrasTest(WriteL1DatasetExtras):
                 extra_name=(
                     DatasetExtraType.dark if task_type == "DARK" else DatasetExtraType.solar_gain
                 ),
-                end_time="2025-01-01T00:00:00",
             )
 
             self.assemble_and_write_dataset_extra(data=data, header=header, filename=filename)
@@ -140,7 +139,6 @@ class ConstructDatasetExtrasTest(WriteL1DatasetExtras):
             total_exposure=0.058,
             readout_exposure=0.029,
             extra_name=DatasetExtraType.demodulation_matrices,
-            end_time="2025-01-01T00:00:00",
         )
         self.assemble_and_write_dataset_extra(data=data, header=header, filename=filename)
 
@@ -217,8 +215,5 @@ def test_construct_dataset_extras(
             assert header["OBSPR_ID"] == "OP1"
             assert header["EXTOBSID"] == "OP2,OP3"
             assert header["EXTNAME"] in ["DARK", "SOLAR GAIN", "DEMODULATION MATRICES"]
-            if header["IPTASK"] == "POLCAL":
-                assert "POLANGLE" not in header
-            else:
-                assert header.get("POLANGLE") == "0.0"
-                assert header.get("RETANGLE") == "0.0"
+            assert header["TELAPSE"] == 3600
+            assert header["DATE-AVG"] == "2023-01-01T00:30:00"

dkist_processing_common/tests/test_trial_catalog.py

@@ -7,8 +7,6 @@ from uuid import uuid4
 import astropy.units as u
 import pytest
 from astropy.io import fits
-from dkist_data_simulator.dataset_extras import DatasetExtraBase
-from dkist_data_simulator.dataset_extras import InstrumentTables
 from dkist_data_simulator.spec214.vbi import SimpleVBIDataset
 from sqids import Sqids
 
@@ -59,6 +57,9 @@ def scratch_with_l1_frames(recipe_run_id, tmp_path) -> WorkflowFileSystem:
         scratch.write(
             file_obj, tags=[Tag.output(), Tag.frame()], relative_path=f"{uuid4().hex}.dat"
         )
+        scratch.write(
+            file_obj, tags=[Tag.output(), Tag.extra()], relative_path=f"{uuid4().hex}.dat"
+        )
 
     return scratch
 

{dkist_processing_common-12.2.0.dist-info → dkist_processing_common-12.2.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 12.2.0
+Version: 12.2.0rc1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-12.2.0.dist-info → dkist_processing_common-12.2.0rc1.dist-info}/RECORD

@@ -1,4 +1,5 @@
 changelog/.gitempty,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+changelog/272.feature.rst,sha256=hqbgG7F741E8KO0k_l9WNFNNi4A5ZYpdABnr34yhbmk,254
 dkist_processing_common/__init__.py,sha256=GQ9EBnYhkOnt-qODclAoLS_g5YVhurxfg1tjVtI9rDI,320
 dkist_processing_common/config.py,sha256=f511KVpK24sQO4dDr4L6PMj5dz0jmWgnx2Y-3DpV0cw,5991
 dkist_processing_common/manual.py,sha256=bIVVyLsbXMh-g_2L3kGROL-1TtJe0_XviHsp7Br31x8,7023
@@ -21,7 +22,7 @@ dkist_processing_common/codecs/str.py,sha256=Xqt5k8IhLc95KiiNiFwB1JWcVVc6T8AfcLr
 dkist_processing_common/fonts/Lato-Regular.ttf,sha256=1jbkaDIx-THtoiLViOlE0IK_0726AvkovuRhwPGFslE,656568
 dkist_processing_common/fonts/__init__.py,sha256=hBvZRtkoGRPlNDWCK-ZePXdSIlThCcjwBDfYaamVgAw,101
 dkist_processing_common/models/__init__.py,sha256=6LMqemdzVZ87fRrpAsbEnTtWZ02_Gu_oajsUlwGRH_Q,74
-dkist_processing_common/models/constants.py,sha256=Xo-HDY-POx9LY5s8lGtXZldrqb1c58t0mpiKKoNdmow,20502
+dkist_processing_common/models/constants.py,sha256=QHLDdMlYab6g5dEivBjkf7O3mW7aJZPSd1OC_EfuRDM,20490
 dkist_processing_common/models/dkist_location.py,sha256=6Nk0wvv4R8ptlrV7BXon7abq4YLvmTdUmPsDN5G8nWc,971
 dkist_processing_common/models/extras.py,sha256=RI4JWOinYl1rRyA4anNDj5nCSIrvwrix_dOod9bcyHA,1207
 dkist_processing_common/models/fits_access.py,sha256=imKqL4-_g6gTR-IeIjZ6qkMhQX3JujdrKFrTd9gOXnw,5605
@@ -60,7 +61,7 @@ dkist_processing_common/parsers/wavelength.py,sha256=P5C9mG8DAKK3GB3vWNRBI5l7pAW
 dkist_processing_common/tasks/__init__.py,sha256=l23ctjNsKJbHbbqaZBMeOPaOtw0hmITEljI_JJ-CVsU,627
 dkist_processing_common/tasks/assemble_movie.py,sha256=1ixDG-f4ODt0vywqVccG3aodLljVO5OGlvuMO9EEvcU,12767
 dkist_processing_common/tasks/base.py,sha256=itAHCvzcodo-q8_AjpWoRaM86BlcjWDpCIiUP7uwmP0,13236
-dkist_processing_common/tasks/l1_output_data.py,sha256=qvOiVyxuqD6efR0PYILERIZ1WPXsJQRIk-hSBf26F5Q,10968
+dkist_processing_common/tasks/l1_output_data.py,sha256=GKmsjJ9FItQqY4ChreiGZVD_gZvCTJF6rzkVyDyaFP8,11499
 dkist_processing_common/tasks/output_data_base.py,sha256=lJZ3olayA_nUWbxzozfbIbJFD3j2VpxPMt1CB9LB5IY,4763
 dkist_processing_common/tasks/parse_l0_input_data.py,sha256=xcWmwSKwxEeManltCFrqVG224Vk-BYRE6g32VOK--rI,19241
 dkist_processing_common/tasks/quality_metrics.py,sha256=cvGF6tJ8yAvxOvkeG3tWxYwL885BrFW5X3V7_MSzL-A,12481
@@ -68,7 +69,7 @@ dkist_processing_common/tasks/teardown.py,sha256=rwT9lWINVDF11-az_nx-Z5ykMTX_SJC
 dkist_processing_common/tasks/transfer_input_data.py,sha256=4TJqlDjTc503QFvzSmMert99r9KHDwyd72r1kHAVhQA,5879
 dkist_processing_common/tasks/trial_catalog.py,sha256=Yf-BKNCT_OHwJsxxZP8p2eRW04CcY0tw5_YIe1e9RQY,10535
 dkist_processing_common/tasks/trial_output_data.py,sha256=pUdrNlAzuir4AUdfax5_MOplB-A9NrXErMJmAwtJmLA,6811
-dkist_processing_common/tasks/write_extra.py,sha256=1mg7AsFHeDEg_ncgGKv57aHixtbzB7jFc5zPLZ3-89Q,16211
+dkist_processing_common/tasks/write_extra.py,sha256=YObiq0xMy5Z1QtpahAKjfELlZ_hcTLtiuD1WKZREeg0,16366
 dkist_processing_common/tasks/write_l1.py,sha256=upsaFN3S0r4MasrzjZ6i0gNF_bvQLf_oyYoRogB7odc,21163
 dkist_processing_common/tasks/write_l1_base.py,sha256=iqejlYb3CSagUyi6U56nmgItzrwcQxLIDwgruxZho3A,2474
 dkist_processing_common/tasks/mixin/__init__.py,sha256=-g-DQbU7m1bclJYuFe3Yh757V-35GIDTbstardKQ7nU,68
@@ -87,7 +88,7 @@ dkist_processing_common/tests/test_assemble_quality.py,sha256=-F22jMY6mPy65VZ1TZ
 dkist_processing_common/tests/test_base.py,sha256=gsyBG2R6Ufx7CzbHeGMagUwM9yCfpN4gCSZ6-aH2q48,6643
 dkist_processing_common/tests/test_codecs.py,sha256=WpF15UYklpNRgETI4EwXsgbNzxMcHlelfpprBbupC0I,23907
 dkist_processing_common/tests/test_constants.py,sha256=I_KcJs7ScCn53GYhEO6qjWrrnfZuyC1IVYOy87Pjlg4,6565
-dkist_processing_common/tests/test_construct_dataset_extras.py,sha256=dss_RXQu0xfOiiuHDCeI0rGrUlDCfgNHKQ7VwkN6HqU,9318
+dkist_processing_common/tests/test_construct_dataset_extras.py,sha256=LjFoLt1oChS298T8vmMK3xT0-5LSxRO3gUexxYSVY7w,9113
 dkist_processing_common/tests/test_cs_step.py,sha256=RA0QD3D8eaL3YSOL_gIJ9wkngy14RQ2jbD-05KAziW4,2408
 dkist_processing_common/tests/test_dkist_location.py,sha256=-_OoSw4SZDLFyIuOltHvM6PQjxm5hTiJQsiTGZ8Sadc,456
 dkist_processing_common/tests/test_fits_access.py,sha256=a50B4IAAH5NH5zeudTqyy0b5uWKJwJuzQLUdK1LoOHM,12832
@@ -112,7 +113,7 @@ dkist_processing_common/tests/test_task_parsing.py,sha256=2_OOmeZQWD17XAd_ECYmod
 dkist_processing_common/tests/test_teardown.py,sha256=DaliHSGsiQBZaFkf5wb3XBo6rHNPmx2bmQtVymYeBN4,5601
 dkist_processing_common/tests/test_transfer_input_data.py,sha256=eyAAWXpTHQ8aew87-MncWpYBn4DAZrTSOL3LvlQfR5Q,12611
 dkist_processing_common/tests/test_transfer_l1_output_data.py,sha256=OV2XMEkCbEuL9_i2S3P9Jfyf15tcBkAP3JNE8Jn_A9k,2137
-dkist_processing_common/tests/test_trial_catalog.py,sha256=290h3cr_ToNtYOm9ze8xQQz9d82sF9XLqmyf9_L7g34,9554
+dkist_processing_common/tests/test_trial_catalog.py,sha256=vnGLV859shiYEP6qOVoYga9vCBMRs6UkNnGtnaps26Q,9548
 dkist_processing_common/tests/test_trial_output_data.py,sha256=fu3iGNV_FI8LOacezyt4HvXnxY3g1_UiBuRI63yz5Oo,11977
 dkist_processing_common/tests/test_workflow_task_base.py,sha256=LTVusltNrsGUOvw9G323am4CXebgE4tJhP6gZCcS0CQ,10457
 dkist_processing_common/tests/test_write_l1.py,sha256=qraecD9Vv6jsqg00XOSlM1IdK9l8i36VoHLoLj8ypiU,27918
@@ -124,7 +125,7 @@ docs/landing_page.rst,sha256=aPAuXFhBx73lEZ59B6E6JXxkK0LlxzD0n-HXqHrfumQ,746
 docs/make.bat,sha256=mBAhtURwhQ7yc95pqwJzlhqBSvRknr1aqZ5s8NKvdKs,4513
 docs/requirements.txt,sha256=Kbl_X4c7RQZw035YTeNB63We6I7pvXFU4T0Uflp2yDY,29
 licenses/LICENSE.rst,sha256=piZaQplkzOMmH1NXg6QIdo9wwo9pPCoHkvm2-DmH76E,1462
-dkist_processing_common-12.2.0.dist-info/METADATA,sha256=z2dP61XdZiapHC3zbjS9kVZncQsd9zkfHd43Aosw4aQ,14211
-dkist_processing_common-12.2.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-dkist_processing_common-12.2.0.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
-dkist_processing_common-12.2.0.dist-info/RECORD,,
+dkist_processing_common-12.2.0rc1.dist-info/METADATA,sha256=jYcIT0L5XRExn309Uky-DKXEKarMZJaHgfu0xF3EXl0,14214
+dkist_processing_common-12.2.0rc1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+dkist_processing_common-12.2.0rc1.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
+dkist_processing_common-12.2.0rc1.dist-info/RECORD,,