dkist-processing-math 2.1.2__py3-none-any.whl → 2.2.0rc1__py3-none-any.whl

changelog/18.misc.rst

@@ -0,0 +1 @@
+Update bitbucket pipelines to use common scripts for checking for changelog snippets and verifying doc builds.

changelog/19.misc.rst

@@ -0,0 +1 @@
+Update Bitbucket pipelines to use execute script for standard steps.

changelog/20.feature.rst

@@ -0,0 +1,2 @@
+Add `stddev_numpy_arrays` function for computing the per-pixel standard deviation of an iterable of arrays.
+If a `generator` is provided this function is very memory-light, even for huge inputs.

dkist_processing_math/statistics.py

@@ -38,3 +38,65 @@ def average_numpy_arrays(arrays: Union[Iterable[np.ndarray], np.ndarray]) -> np.
     if output is not None:
         return output / (count + 1)
     raise ValueError("data_arrays is empty")
+
+
+def stddev_numpy_arrays(arrays: Iterable[np.ndarray]) -> np.ndarray:
+    r"""
+    Given an iterable of numpy arrays, calculate the pixel-wise standard deviation and return it in a numpy array.
+
+    This function uses Welford's single-pass algorithm. The standard deviation for the :math:`n^{th}` array is
+
+    .. math::
+        \sigma_n = \sqrt{\frac{M_{2,n}}{n}}
+
+    where
+
+    .. math::
+        M_{2,n} = M_{2,n-1} + (x_n - \bar{x}_{n-1})(x_n - \bar{x}_n)
+
+    and
+
+    .. math::
+        \bar{x}_n = \bar{x}_{n-1} + \frac{x_n - \bar{x}_{n-1}}{n}
+
+    Note that in computing :math:`\sigma` we divide by :math:`n` instead of :math:`(n-1)`. This is to match the default
+    behavior of `numpy.std`.
+
+    Parameters
+    ----------
+    arrays
+        The arrays to be to compute the standard deviation of
+
+    Returns
+    -------
+    The standard deviation of the input arrays
+    """
+    if isinstance(arrays, np.ndarray):
+        raise ValueError(
+            "Need an Iterator with at least two arrays, but a single array was provided."
+        )
+
+    running_mean = 0.0
+    running_M2 = 0.0
+    count = 0
+    for array in arrays:
+        count += 1
+        try:
+            delta = array - running_mean
+        except ValueError:
+            raise ValueError(
+                f"All arrays must be the same shape. "
+                f"Shape of initial array = {running_mean.shape} "
+                f"Shape of current array = {array.shape}"
+            )
+        running_mean += delta / count
+        delta2 = array - running_mean
+        running_M2 += delta * delta2
+
+    match count:
+        case 0:
+            raise ValueError("arrays is empty")
+        case 1:
+            raise ValueError(f"Need at least two arrays. Provided Iterator only has {count}")
+        case _:
+            return np.sqrt(running_M2 / count)

dkist_processing_math/tests/conftest.py

@@ -4,8 +4,13 @@ import pytest
 rng = np.random.default_rng()
 
 
+@pytest.fixture
+def test_array_shape() -> tuple[int, int]:
+    return (10, 10)
+
+
 @pytest.fixture()
-def numpy_arrays():
+def numpy_arrays(test_array_shape):
     """
     Create an iterable of random 10 x 10 arrays
 
@@ -14,7 +19,11 @@ def numpy_arrays():
     List[np.ndarray]
 
     """
-    arrays = [rng.standard_normal((10, 10)), rng.standard_normal((10, 10))]
+    arrays = [
+        rng.standard_normal(test_array_shape),
+        rng.standard_normal(test_array_shape),
+        rng.standard_normal(test_array_shape),
+    ]
     return arrays
 
 
@@ -32,7 +41,7 @@ def numpy_array(numpy_arrays):
 
 
 @pytest.fixture()
-def numpy_arrays_wrong_shape():
+def numpy_arrays_wrong_shape(test_array_shape):
     """
     Create an iterable of random arrays of differing sizes
 
@@ -41,7 +50,10 @@ def numpy_arrays_wrong_shape():
     List[np.ndarray]
 
     """
-    arrays = [rng.standard_normal((10, 10)), rng.standard_normal((10, 15))]
+    arrays = [
+        rng.standard_normal(test_array_shape),
+        rng.standard_normal(tuple(i * 2 for i in test_array_shape)),
+    ]
     return arrays
 
 

dkist_processing_math/tests/test_statistics.py

@@ -2,6 +2,7 @@ import numpy as np
 import pytest
 
 from dkist_processing_math.statistics import average_numpy_arrays
+from dkist_processing_math.statistics import stddev_numpy_arrays
 
 
 @pytest.fixture()
@@ -60,3 +61,24 @@ def test_average_with_large_values(int_arrays_with_large_values):
     """
     expected = np.ones((10, 10)) * 17500
     np.testing.assert_equal(expected, average_numpy_arrays(int_arrays_with_large_values))
+
+
+def test_stddev_numpy_arrays(numpy_arrays):
+    expected_result = np.std(numpy_arrays, axis=0)
+    result = stddev_numpy_arrays(numpy_arrays)
+
+    np.testing.assert_allclose(result, expected_result)
+
+
+def test_stddec_numpy_arrays_failures(numpy_arrays, numpy_arrays_wrong_shape):
+    with pytest.raises(ValueError, match="Need an Iterator with at least two arrays"):
+        stddev_numpy_arrays(numpy_arrays[0])
+
+    with pytest.raises(ValueError, match="All arrays must be the same shape"):
+        stddev_numpy_arrays(numpy_arrays_wrong_shape)
+
+    with pytest.raises(ValueError, match="Need at least two arrays"):
+        stddev_numpy_arrays([numpy_arrays[0]])
+
+    with pytest.raises(ValueError, match="arrays is empty"):
+        stddev_numpy_arrays([])

{dkist_processing_math-2.1.2.dist-info → dkist_processing_math-2.2.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: dkist-processing-math
-Version: 2.1.2
+Version: 2.2.0rc1
 Summary: Common math functionality used in the DKIST calibrations suite
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_math-2.1.2.dist-info → dkist_processing_math-2.2.0rc1.dist-info}/RECORD

@@ -1,15 +1,18 @@
 changelog/.gitempty,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+changelog/18.misc.rst,sha256=nhzzLztW_sNOvZ0EeO0jCOpMf1StXdl6IpsJWS6sc4I,111
+changelog/19.misc.rst,sha256=oCFAWH7tQxLKxn6VfQBzAocWj903IvwXlgotpVuRSXY,69
+changelog/20.feature.rst,sha256=5gRsTu_0ql426Z1Wn-x9IXiALTiIZYpczx3_9Y-HYR4,195
 dkist_processing_math/__init__.py,sha256=Mf7liCQC9M5YgFrz2NZ0dZ5smvcuw47ww6MC9mijAHw,249
 dkist_processing_math/arithmetic.py,sha256=h8E4DUBGQLDUiNWmZb2kZRrLMmjEpdl1S_mIi3ntZ6w,3694
 dkist_processing_math/feature.py,sha256=vwrkXBlK7wM6-KFaWxhBsMpHiHbpsWOiuGOhf3PRLSY,2003
 dkist_processing_math/linear_algebra.py,sha256=3ycmWklgLI89NjQgwgY81Zv1m_L4jJXHnt70TzQDVWQ,4179
-dkist_processing_math/statistics.py,sha256=Dgt1gSDQDN2XGV38xSQ3ZUmyIeGMZRoBl2vGHcADAYs,1233
+dkist_processing_math/statistics.py,sha256=_N6LPRTAFT6lQXv8DJtzK6icIHT3JsM4YWR7JfOSiHA,3058
 dkist_processing_math/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dkist_processing_math/tests/conftest.py,sha256=Pg7lDltF_N7SVIjAi0rRADXBftsSzw0o94-SfKW1Zko,1333
+dkist_processing_math/tests/conftest.py,sha256=TV0o-R2lU1MvZleme0w-akcgO5wOTlFNKhjSlmDwkqg,1593
 dkist_processing_math/tests/test_arithmetic.py,sha256=18YI4Fr_m0KU0dYR8_ef7CMmnOlb-Zc4I8ERgoYdxdA,5658
 dkist_processing_math/tests/test_feature.py,sha256=ZB8r_IUrAyTAlKbxEXwr8Q2zS7KPmlJhaU1l0guK0Xc,2311
 dkist_processing_math/tests/test_linear_algebra.py,sha256=k-XqIIBPsenNMKqo2nQnDkkN-6T1nvVT8BrJnupAIFU,6495
-dkist_processing_math/tests/test_statistics.py,sha256=0_bFuKxcu6hfimScB5du38qzABq2syX7SQrA9ZjwbuA,2173
+dkist_processing_math/tests/test_statistics.py,sha256=N0S8C9Bwhd1VgCAkm6RqIZF_Ulwrcet9wqfbndw8KZs,2998
 dkist_processing_math/tests/test_transform.py,sha256=M761Yceh3FKV00udvb6_AeKuyBpQrUAG-mLfO-PbHmc,20253
 dkist_processing_math/transform/__init__.py,sha256=gZGyavJgtuquv8vc_VkXraJjz_8MkDTk9UwGQhthHwM,617
 dkist_processing_math/transform/affine.py,sha256=wJvL-R3DMpOmEnCKL8YI8eRUE7enH2Vg-mSLLpfCEoE,8595
@@ -24,7 +27,7 @@ docs/landing_page.rst,sha256=JCyR9SRhYS6zeEUlonIzko1kQ0tAFDIfi7pCoQanZeA,251
 docs/make.bat,sha256=mBAhtURwhQ7yc95pqwJzlhqBSvRknr1aqZ5s8NKvdKs,4513
 docs/requirements.txt,sha256=Kbl_X4c7RQZw035YTeNB63We6I7pvXFU4T0Uflp2yDY,29
 licenses/LICENSE.rst,sha256=piZaQplkzOMmH1NXg6QIdo9wwo9pPCoHkvm2-DmH76E,1462
-dkist_processing_math-2.1.2.dist-info/METADATA,sha256=lK_TFbwqPyS8_FCZ2dKcmmpqy1tDIXJcCo-QESZxpbI,4284
-dkist_processing_math-2.1.2.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-dkist_processing_math-2.1.2.dist-info/top_level.txt,sha256=7NeFrOlR4xhZ3LMT4jN99LQugukjr6l9u3w_s6b-uO4,46
-dkist_processing_math-2.1.2.dist-info/RECORD,,
+dkist_processing_math-2.2.0rc1.dist-info/METADATA,sha256=20cGjV34kHUjkZI-Nm67oi8Y6VbYUg2_V1ELyN_kl08,4287
+dkist_processing_math-2.2.0rc1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+dkist_processing_math-2.2.0rc1.dist-info/top_level.txt,sha256=7NeFrOlR4xhZ3LMT4jN99LQugukjr6l9u3w_s6b-uO4,46
+dkist_processing_math-2.2.0rc1.dist-info/RECORD,,

{dkist_processing_math-2.1.2.dist-info → dkist_processing_math-2.2.0rc1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any