dkist-processing-common 6.2.2__tar.gz → 6.2.3rc1__tar.gz

{dkist-processing-common-6.2.2 → dkist-processing-common-6.2.3rc1}/CHANGELOG.rst

@@ -7,6 +7,12 @@ Features
 - Add the ability to create a quality report from a trial workflow. (`#185 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/185>`__)
 
 
+Bugfixes
+--------
+
+- `QualityL0Metrics.calculate_l0_metrics` now correctly identifies the TASK type. Previously it could have erroneously used the WORKFLOWTASK tag to find the IP TASK TYPE. (`#185 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/185>`__)
+
+
 v6.2.1 (2024-05-01)
 ===================
 

{dkist-processing-common-6.2.2 → dkist-processing-common-6.2.3rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-common
-Version: 6.2.2
+Version: 6.2.3rc1
 Summary: Common task classes used by the DKIST Science Data Processing pipelines to process DKIST data.
 Home-page: https://bitbucket.org/dkistdc/dkist-processing-common/src/main/
 Author: NSO / AURA

dkist-processing-common-6.2.3rc1/changelog/189.misc.rst

@@ -0,0 +1 @@
+Cap the length of browse movies at 60 seconds by default.

dkist-processing-common-6.2.3rc1/changelog/190.bugfix.rst

@@ -0,0 +1 @@
+`QualityMixin.avg_noise` is now NaN aware. I.e., it will ignore NaN values when computing the noise.

dkist-processing-common-6.2.3rc1/changelog/191.feature.rst

@@ -0,0 +1 @@
+Save all floating point arrays as float32. The extra precision of float64 is not needed, especially when lossy quantization is applied before compression.

{dkist-processing-common-6.2.2 → dkist-processing-common-6.2.3rc1}/dkist_processing_common/tasks/assemble_movie.py

@@ -64,6 +64,7 @@ class AssembleMovie(WorkflowTaskBase, ABC):
     MOVIE_FRAME_SHAPE = (2048, 1536)  # Is this a/THE standard size?
     IMAGES_PER_SEC = 3
     MINIMUM_DURATION = 10  # seconds
+    MAXIMUM_DURATION = 60  # seconds
     FPS = 15
     FONT_FILE = pkg_resources.resource_filename("dkist_processing_common", "fonts/Lato-Regular.ttf")
     TEXT_MARGIN_PX = 5
@@ -126,6 +127,10 @@ class AssembleMovie(WorkflowTaskBase, ABC):
             self.IMAGES_PER_SEC = self.num_images / self.MINIMUM_DURATION
             self.FPS = round(self.num_images / self.MINIMUM_DURATION) or 1
 
+        # IF movie length would be more than maximum duration, stride the frames
+        if (self.num_images / self.IMAGES_PER_SEC) > self.MAXIMUM_DURATION:
+            self.IMAGES_PER_SEC = self.num_images / self.MAXIMUM_DURATION
+
         self.duration = self.num_images / self.IMAGES_PER_SEC
 
         ## This is here to fix a floating point bug in moviepy

{dkist-processing-common-6.2.2 → dkist-processing-common-6.2.3rc1}/dkist_processing_common/tasks/mixin/quality/_base.py

@@ -133,12 +133,12 @@ class QualityMixin(
 
             square_4 = data[-corner_square_length:, -corner_square_height:]  # bottom right
 
-            return np.average(
+            return np.nanmean(
                 [
-                    np.std(square_1),
-                    np.std(square_2),
-                    np.std(square_3),
-                    np.std(square_4),
+                    np.nanstd(square_1),
+                    np.nanstd(square_2),
+                    np.nanstd(square_3),
+                    np.nanstd(square_4),
                 ]
             )
 
@@ -179,15 +179,15 @@ class QualityMixin(
                 -corner_cube_length:, -corner_cube_height:, -corner_cube_width:
             ]  # bottom right back
 
-            return np.average(
+            return np.nanmean(
                 [
-                    np.std(cube_1),
-                    np.std(cube_2),
-                    np.std(cube_3),
-                    np.std(cube_4),
-                    np.std(cube_5),
-                    np.std(cube_6),
-                    np.std(cube_7),
-                    np.std(cube_8),
+                    np.nanstd(cube_1),
+                    np.nanstd(cube_2),
+                    np.nanstd(cube_3),
+                    np.nanstd(cube_4),
+                    np.nanstd(cube_5),
+                    np.nanstd(cube_6),
+                    np.nanstd(cube_7),
+                    np.nanstd(cube_8),
                 ]
             )

{dkist-processing-common-6.2.2 → dkist-processing-common-6.2.3rc1}/dkist_processing_common/tasks/write_l1.py

@@ -68,12 +68,17 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
                     stokes_param=stokes_param,
                 )
 
+                data_array = calibrated_fits_object.data
+                # Cast array to float32 if float64
+                if np.issubdtype(data_array.dtype, np.float64):
+                    # Cast to float32 with the conservative casting option
+                    # just incase something weird has happened.
+                    data_array = data_array.astype(np.float32, casting="same_kind")
+
                 # Get the tile size to use for compression. None means use astropy defaults
-                tile_size = self._get_tile_size(calibrated_fits_object.data)
+                tile_size = self._get_tile_size(data_array)
                 # Write frame to disk - compressed
-                hdu = fits.CompImageHDU(
-                    header=l1_header, data=calibrated_fits_object.data, tile_shape=tile_size
-                )
+                hdu = fits.CompImageHDU(header=l1_header, data=data_array, tile_shape=tile_size)
                 formatted_header = reformat_spec214_header(hdu.header)
                 hdu = fits.CompImageHDU(
                     header=formatted_header, data=hdu.data, tile_shape=tile_size

{dkist-processing-common-6.2.2 → dkist-processing-common-6.2.3rc1}/dkist_processing_common/tests/test_quality_mixin.py

@@ -974,3 +974,23 @@ def test_polcal_build_efficiency(quality_task, polcal_efficiency_json):
     assert metric["name"] == f"PolCal Modulation Efficiency - {label}"
     efficiency_data = metric["efficiency_data"]
     assert efficiency_data["efficiency_list"] == [[1.0, 2.0], [2.0, 3.0]]
+
+
+@pytest.mark.parametrize(
+    "array_shape", [pytest.param((100, 100), id="2D"), pytest.param((100, 100, 100), id="3D")]
+)
+def test_avg_noise_nan_values(quality_task, array_shape):
+    """
+    Given: A data array that contains NaN values
+    When: Computing the noise with `avg_noise`
+    Then: A non-NaN value is returned
+    """
+    rng = np.random.default_rng()
+    data = rng.random(array_shape)
+
+    # Make 1/3 of the px NaNs
+    nan_idx = np.where((data > 0.333) & (data < 0.666))
+    data[nan_idx] = np.nan
+    result = quality_task.avg_noise(data)
+
+    assert not np.isnan(result)

{dkist-processing-common-6.2.2 → dkist-processing-common-6.2.3rc1}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-common
-Version: 6.2.2
+Version: 6.2.3rc1
 Summary: Common task classes used by the DKIST Science Data Processing pipelines to process DKIST data.
 Home-page: https://bitbucket.org/dkistdc/dkist-processing-common/src/main/
 Author: NSO / AURA

{dkist-processing-common-6.2.2 → dkist-processing-common-6.2.3rc1}/dkist_processing_common.egg-info/SOURCES.txt

@@ -9,6 +9,9 @@ pyproject.toml
 setup.cfg
 setup.py
 changelog/.gitempty
+changelog/189.misc.rst
+changelog/190.bugfix.rst
+changelog/191.feature.rst
 dkist_processing_common/__init__.py
 dkist_processing_common/manual.py
 dkist_processing_common.egg-info/PKG-INFO