dkist-processing-common 12.3.0__tar.gz → 12.4.0__tar.gz

{dkist_processing_common-12.3.0 → dkist_processing_common-12.4.0}/CHANGELOG.rst

@@ -1,3 +1,14 @@
+v12.4.0 (2026-02-04)
+====================
+
+Misc
+----
+
+- When calculating the updated frame volume for the `FRAMEVOL` key, don't decompress the file. (`#292 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/292>`__)
+- Refactor calculation of statistics headers for improved performance. (`#293 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/293>`__)
+- Send the L1 header for validation instead of the whole file for performance improvements. (`#294 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/294>`__)
+
+
 v12.3.0 (2026-02-02)
 ====================
 

{dkist_processing_common-12.3.0 → dkist_processing_common-12.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 12.3.0
+Version: 12.4.0
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-12.3.0 → dkist_processing_common-12.4.0}/dkist_processing_common/tasks/write_extra.py

@@ -320,7 +320,10 @@ class WriteL1DatasetExtras(WriteL1Base, ABC):
 
         # Check that the written file passes spec 214 validation if requested
         if self.validate_l1_on_write:
-            spec_extras_validator.validate(self.scratch.absolute_path(filename), extra=False)
+            # Replace this logic with the fits_header_decoder once FITS library keys have been removed from the SPEC
+            with fits.open(self.scratch.absolute_path(filename)) as hdul:
+                header = hdul[1].header
+                spec_extras_validator.validate(header, extra=False)
 
     @abstractmethod
     def run(self) -> None:

{dkist_processing_common-12.3.0 → dkist_processing_common-12.4.0}/dkist_processing_common/tasks/write_l1.py

@@ -27,6 +27,7 @@ from sunpy.coordinates import Helioprojective
 
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
+from dkist_processing_common.codecs.fits import fits_header_decoder
 from dkist_processing_common.models.dkist_location import location_of_dkist
 from dkist_processing_common.models.fried_parameter import r0_valid
 from dkist_processing_common.models.tags import Tag
@@ -102,9 +103,10 @@ class WriteL1Frame(WriteL1Base, ABC):
 
                 # Check that the written file passes spec 214 validation if requested
                 if self.validate_l1_on_write:
-                    spec214_validator.validate(
-                        self.scratch.absolute_path(relative_path), extra=False
-                    )
+                    # Replace this logic with the fits_header_decoder once FITS library keys have been removed from the SPEC
+                    with fits.open(self.scratch.absolute_path(relative_path)) as hdul:
+                        header = hdul[1].header
+                        spec214_validator.validate(header, extra=False)
 
     def replace_header_values(self, header: fits.Header, data: np.ndarray) -> fits.Header:
         """Replace header values that should already exist with new values."""
@@ -132,25 +134,27 @@ class WriteL1Frame(WriteL1Base, ABC):
     @staticmethod
     def add_stats_headers(header: fits.Header, data: np.ndarray) -> fits.Header:
         """Fill out the spec 214 statistics header table."""
-        data = data.flatten()
-        percentiles = np.nanpercentile(data, [1, 2, 5, 10, 25, 75, 90, 95, 98, 99])
-        header["DATAMIN"] = np.nanmin(data)
-        header["DATAMAX"] = np.nanmax(data)
-        header["DATAMEAN"] = np.nanmean(data)
-        header["DATAMEDN"] = np.nanmedian(data)
+        flat_data = np.ravel(data)
+        nan_mask = np.isfinite(flat_data)
+        valid_data = flat_data[nan_mask]
+        percentiles = np.percentile(valid_data, [1, 2, 5, 10, 25, 50, 75, 90, 95, 98, 99])
+        header["DATAMIN"] = valid_data.min()
+        header["DATAMAX"] = valid_data.max()
+        header["DATAMEAN"] = valid_data.mean()
+        header["DATAMEDN"] = percentiles[5]
         header["DATAP01"] = percentiles[0]
         header["DATAP02"] = percentiles[1]
         header["DATAP05"] = percentiles[2]
         header["DATAP10"] = percentiles[3]
         header["DATAP25"] = percentiles[4]
-        header["DATAP75"] = percentiles[5]
-        header["DATAP90"] = percentiles[6]
-        header["DATAP95"] = percentiles[7]
-        header["DATAP98"] = percentiles[8]
-        header["DATAP99"] = percentiles[9]
-        header["DATARMS"] = np.sqrt(np.nanmean(data**2))
-        header["DATAKURT"] = kurtosis(data, nan_policy="omit")
-        header["DATASKEW"] = skew(data, nan_policy="omit")
+        header["DATAP75"] = percentiles[6]
+        header["DATAP90"] = percentiles[7]
+        header["DATAP95"] = percentiles[8]
+        header["DATAP98"] = percentiles[9]
+        header["DATAP99"] = percentiles[10]
+        header["DATARMS"] = np.sqrt(np.dot(valid_data, valid_data) / valid_data.size)
+        header["DATAKURT"] = kurtosis(valid_data)
+        header["DATASKEW"] = skew(valid_data)
         return header
 
     def add_datacenter_headers(

{dkist_processing_common-12.3.0 → dkist_processing_common-12.4.0}/dkist_processing_common/tasks/write_l1_base.py

@@ -38,7 +38,7 @@ class WriteL1Base(WorkflowTaskBase, MetadataStoreMixin, ABC):
         """Update FRAMEVOL key to be exactly the size of the file on-disk."""
         full_path = self.scratch.workflow_base_path / relative_path
         compressed_size = full_path.stat().st_size / 1024 / 1024
-        hdul = fits.open(full_path, mode="update")
+        hdul = fits.open(full_path, mode="update", disable_image_compression=True)
         for i in range(1, len(hdul)):
             hdul[i].header["FRAMEVOL"] = compressed_size
         hdul.flush()

{dkist_processing_common-12.3.0 → dkist_processing_common-12.4.0}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 12.3.0
+Version: 12.4.0
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause