dkist-processing-common 12.2.0__tar.gz → 12.2.0rc1__tar.gz

{dkist_processing_common-12.2.0 → dkist_processing_common-12.2.0rc1}/CHANGELOG.rst

@@ -1,12 +1,3 @@
-v12.2.0 (2026-01-30)
-====================
-
-Features
---------
-
-- Add the framework for using Dataset Extras, that is, other data that can be included with the L1 FITS files generated by the regular pipeline. A new abstract class, WriteL1DatasetExtras, provides helper functionality for use in the instrument pipelines. (`#272 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/272>`__)
-
-
 v12.1.0 (2026-01-26)
 ====================
 

{dkist_processing_common-12.2.0 → dkist_processing_common-12.2.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 12.2.0
+Version: 12.2.0rc1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

dkist_processing_common-12.2.0rc1/changelog/272.feature.rst

@@ -0,0 +1 @@
+Add the framework for using Dataset Extras, that is, other data that can be included with the L1 FITS files generated by the regular pipeline. A new abstract class, WriteL1DatasetExtras, provides helper functionality for use in the instrument pipelines.

{dkist_processing_common-12.2.0 → dkist_processing_common-12.2.0rc1}/dkist_processing_common/models/constants.py

@@ -473,14 +473,14 @@ class ConstantsBase:
         return self._db_dict[BudName.solar_gain_gos_polarizer_status]
 
     @property
-    def dark_gos_polarizer_angle(self) -> str:
+    def dark_gos_polarizer_angle(self) -> float:
         """Return the gos polarizer angle constant for the dark task."""
-        return str(self._db_dict[BudName.dark_gos_polarizer_angle])
+        return self._db_dict[BudName.dark_gos_polarizer_angle]
 
     @property
-    def solar_gain_gos_polarizer_angle(self) -> str:
+    def solar_gain_gos_polarizer_angle(self) -> float:
         """Return the gos polarizer angle constant for the solar gain task."""
-        return str(self._db_dict[BudName.solar_gain_gos_polarizer_angle])
+        return self._db_dict[BudName.solar_gain_gos_polarizer_angle]
 
     @property
     def dark_gos_retarder_status(self) -> str:
@@ -493,14 +493,14 @@ class ConstantsBase:
         return self._db_dict[BudName.solar_gain_gos_retarder_status]
 
     @property
-    def dark_gos_retarder_angle(self) -> str:
+    def dark_gos_retarder_angle(self) -> float:
         """Return the gos retarder angle constant for the dark task."""
-        return str(self._db_dict[BudName.dark_gos_retarder_angle])
+        return self._db_dict[BudName.dark_gos_retarder_angle]
 
     @property
-    def solar_gain_gos_retarder_angle(self) -> str:
+    def solar_gain_gos_retarder_angle(self) -> float:
         """Return the gos retarder angle constant for the solar gain task."""
-        return str(self._db_dict[BudName.solar_gain_gos_retarder_angle])
+        return self._db_dict[BudName.solar_gain_gos_retarder_angle]
 
     @property
     def dark_gos_level0_status(self) -> str:

{dkist_processing_common-12.2.0 → dkist_processing_common-12.2.0rc1}/dkist_processing_common/tasks/l1_output_data.py

@@ -2,14 +2,18 @@
 
 import logging
 from abc import ABC
+from itertools import chain
 from pathlib import Path
 from typing import Iterable
 
+from dkist_processing_common.codecs.quality import quality_data_decoder
 from dkist_processing_common.codecs.quality import quality_data_encoder
 from dkist_processing_common.models.message import CatalogFrameMessage
 from dkist_processing_common.models.message import CatalogFrameMessageBody
 from dkist_processing_common.models.message import CatalogObjectMessage
 from dkist_processing_common.models.message import CatalogObjectMessageBody
+from dkist_processing_common.models.message import CreateQualityReportMessage
+from dkist_processing_common.models.message import CreateQualityReportMessageBody
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.mixin.globus import GlobusMixin
 from dkist_processing_common.tasks.mixin.interservice_bus import InterserviceBusMixin
@@ -189,9 +193,7 @@ class SubmitDatasetMetadata(L1OutputDataBase):
 class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
     """Task class for publishing Catalog and Quality Messages."""
 
-    def frame_messages(
-        self, paths: Iterable[Path], folder_modifier: str | None = None
-    ) -> list[CatalogFrameMessage]:
+    def frame_messages(self, paths: Iterable[Path]) -> list[CatalogFrameMessage]:
         """
         Create the frame messages.
 
@@ -199,8 +201,6 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         ----------
         paths
             The input paths for which to publish frame messages
-        folder_modifier
-            A subdirectory to use if the files in paths are not in the base directory
 
         Returns
         -------
@@ -208,7 +208,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         """
         message_bodies = [
             CatalogFrameMessageBody(
-                objectName=self.format_object_key(path=p, folder_modifier=folder_modifier),
+                objectName=self.format_object_key(path=p),
                 conversationId=str(self.recipe_run_id),
                 bucket=self.destination_bucket,
             )
@@ -237,7 +237,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         message_bodies = [
             CatalogObjectMessageBody(
                 objectType=object_type,
-                objectName=self.format_object_key(path=p),
+                objectName=self.format_object_key(p),
                 bucket=self.destination_bucket,
                 conversationId=str(self.recipe_run_id),
                 groupId=self.constants.dataset_id,
@@ -260,7 +260,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
             messages = []
             messages += self.frame_messages(paths=frames)
             frame_message_count = len(messages)
-            messages += self.frame_messages(paths=extras, folder_modifier="extra")
+            messages += self.frame_messages(paths=extras)
             extra_message_count = len(extras)
             messages += self.object_messages(paths=movies, object_type="MOVIE")
             object_message_count = len(movies)
@@ -270,4 +270,14 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
         with self.telemetry_span(
             f"Publish messages: {frame_message_count = }, {extra_message_count = }, {object_message_count = }, {dataset_has_quality_data = }"
         ):
+            messages.append(self.quality_report_message)
+        with self.telemetry_span("Publish Catalog Messages") as publish_span:
+            publish_span.set_attributes(
+                {
+                    "frame_message_count": frame_message_count,
+                    "extra_message_count": extra_message_count,
+                    "object_message_count": object_message_count,
+                    "dataset_has_quality_data": dataset_has_quality_data,
+                }
+            )
             self.interservice_bus_publish(messages=messages)

{dkist_processing_common-12.2.0 → dkist_processing_common-12.2.0rc1}/dkist_processing_common/tasks/write_extra.py

@@ -27,7 +27,6 @@ class WriteL1DatasetExtras(WriteL1Base, ABC):
         filename: str,
         task_type: TaskName,
         extra_name: DatasetExtraType,
-        end_time: str,
         total_exposure: float | None = None,
         readout_exposure: float | None = None,
     ) -> dict:
@@ -39,6 +38,7 @@ class WriteL1DatasetExtras(WriteL1Base, ABC):
                     self.constants.dark_observing_program_execution_ids
                 )
                 task_specific_date_begin = self.constants.dark_date_begin
+                task_specific_date_end = self.constants.dark_date_end
                 task_specific_raw_frames_per_fpa = (
                     0  # can be updated in construction of dataset extra if required
                 )
@@ -74,6 +74,7 @@ class WriteL1DatasetExtras(WriteL1Base, ABC):
                 )
 
                 task_specific_date_begin = self.constants.solar_gain_date_begin
+                task_specific_date_end = self.constants.solar_gain_date_end
                 task_specific_raw_frames_per_fpa = self.constants.solar_gain_num_raw_frames_per_fpa
                 task_specific_telescope_tracking_mode = (
                     self.constants.solar_gain_telescope_tracking_mode
@@ -109,6 +110,7 @@ class WriteL1DatasetExtras(WriteL1Base, ABC):
                 )
 
                 task_specific_date_begin = self.constants.polcal_date_begin
+                task_specific_date_end = self.constants.polcal_date_end
                 task_specific_raw_frames_per_fpa = self.constants.polcal_num_raw_frames_per_fpa
                 task_specific_telescope_tracking_mode = (
                     self.constants.polcal_telescope_tracking_mode
@@ -140,14 +142,14 @@ class WriteL1DatasetExtras(WriteL1Base, ABC):
                 raise ValueError(f"Unsupported task type {task_type}")
 
         start_datetime = datetime.fromisoformat(task_specific_date_begin)
-        end_datetime = datetime.fromisoformat(end_time)
+        end_datetime = datetime.fromisoformat(task_specific_date_end)
 
         dataset_extra_header = {
             DatasetExtraHeaderSection.common: {
                 "BUNIT": "count",
                 "DATE": Time.now().fits,
                 "DATE-BEG": task_specific_date_begin,
-                "DATE-END": end_time,
+                "DATE-END": task_specific_date_end,
                 "TELAPSE": (end_datetime - start_datetime).total_seconds(),
                 "DATE-AVG": (start_datetime + (end_datetime - start_datetime) / 2).isoformat(),
                 "TIMESYS": "UTC",
@@ -263,7 +265,6 @@ class WriteL1DatasetExtras(WriteL1Base, ABC):
         extra_name: DatasetExtraType,
         total_exposure: float | None = None,
         readout_exposure: float | None = None,
-        end_time: str | None = None,
     ) -> fits.Header:
         """Build FITS header for dataset extra file."""
         header = fits.Header()
@@ -273,7 +274,6 @@ class WriteL1DatasetExtras(WriteL1Base, ABC):
             total_exposure=total_exposure,
             readout_exposure=readout_exposure,
             extra_name=extra_name,
-            end_time=end_time,
         )
         for section in sections:
             header.update(all_section_headers[section].items())

{dkist_processing_common-12.2.0 → dkist_processing_common-12.2.0rc1}/dkist_processing_common/tests/test_construct_dataset_extras.py

@@ -118,7 +118,6 @@ class ConstructDatasetExtrasTest(WriteL1DatasetExtras):
                 extra_name=(
                     DatasetExtraType.dark if task_type == "DARK" else DatasetExtraType.solar_gain
                 ),
-                end_time="2025-01-01T00:00:00",
             )
 
             self.assemble_and_write_dataset_extra(data=data, header=header, filename=filename)
@@ -140,7 +139,6 @@ class ConstructDatasetExtrasTest(WriteL1DatasetExtras):
             total_exposure=0.058,
             readout_exposure=0.029,
             extra_name=DatasetExtraType.demodulation_matrices,
-            end_time="2025-01-01T00:00:00",
         )
         self.assemble_and_write_dataset_extra(data=data, header=header, filename=filename)
 
@@ -217,8 +215,5 @@ def test_construct_dataset_extras(
             assert header["OBSPR_ID"] == "OP1"
             assert header["EXTOBSID"] == "OP2,OP3"
             assert header["EXTNAME"] in ["DARK", "SOLAR GAIN", "DEMODULATION MATRICES"]
-            if header["IPTASK"] == "POLCAL":
-                assert "POLANGLE" not in header
-            else:
-                assert header.get("POLANGLE") == "0.0"
-                assert header.get("RETANGLE") == "0.0"
+            assert header["TELAPSE"] == 3600
+            assert header["DATE-AVG"] == "2023-01-01T00:30:00"

{dkist_processing_common-12.2.0 → dkist_processing_common-12.2.0rc1}/dkist_processing_common/tests/test_trial_catalog.py

@@ -7,8 +7,6 @@ from uuid import uuid4
 import astropy.units as u
 import pytest
 from astropy.io import fits
-from dkist_data_simulator.dataset_extras import DatasetExtraBase
-from dkist_data_simulator.dataset_extras import InstrumentTables
 from dkist_data_simulator.spec214.vbi import SimpleVBIDataset
 from sqids import Sqids
 
@@ -59,6 +57,9 @@ def scratch_with_l1_frames(recipe_run_id, tmp_path) -> WorkflowFileSystem:
         scratch.write(
             file_obj, tags=[Tag.output(), Tag.frame()], relative_path=f"{uuid4().hex}.dat"
         )
+        scratch.write(
+            file_obj, tags=[Tag.output(), Tag.extra()], relative_path=f"{uuid4().hex}.dat"
+        )
 
     return scratch
 

{dkist_processing_common-12.2.0 → dkist_processing_common-12.2.0rc1}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 12.2.0
+Version: 12.2.0rc1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-12.2.0 → dkist_processing_common-12.2.0rc1}/dkist_processing_common.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ README.rst
 bitbucket-pipelines.yml
 pyproject.toml
 changelog/.gitempty
+changelog/272.feature.rst
 dkist_processing_common/__init__.py
 dkist_processing_common/config.py
 dkist_processing_common/manual.py