dkist-processing-math 2.1.2__tar.gz → 2.2.0rc1__tar.gz

{dkist_processing_math-2.1.2 → dkist_processing_math-2.2.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: dkist-processing-math
-Version: 2.1.2
+Version: 2.2.0rc1
 Summary: Common math functionality used in the DKIST calibrations suite
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

dkist_processing_math-2.2.0rc1/bitbucket-pipelines.yml

@@ -0,0 +1,72 @@
+image: python:3.11
+definitions:
+  steps:
+    - step: &download_execute_script
+        name: Download Script Execute Script
+        script:
+          - echo "Retrieving Script Execution Script"
+          - curl -fL https://getcli.jfrog.io | sh
+          - ./jfrog rt dl --url $ARTIFACTORY_URL --user $ARTIFACTORY_USER --password $ARTIFACTORY_PASSWORD generic-packages/ci_scripts/latest/execute_script.sh
+          - cat ci_scripts/latest/execute_script.sh
+          - chmod 755 ci_scripts/latest/execute_script.sh
+        artifacts:
+          - ci_scripts/latest/execute_script.sh
+    - step: &lint
+        caches:
+          - pip
+        name: Lint
+        script:
+          - ./ci_scripts/latest/execute_script.sh lint_python.sh
+    - step: &changelog
+        caches:
+          - pip
+        name: Changelog
+        script:
+          - ./ci_scripts/latest/execute_script.sh check_changelog.sh
+    - step: &scan
+        caches:
+          - pip
+        name: Scan
+        script:
+          - ./ci_scripts/latest/execute_script.sh scan_python.sh
+    - step: &test
+        caches:
+          - pip
+        name: Test
+        script:
+          - pip install .[test]
+          - pytest --cov -m "not development" dkist_processing_math
+    - step: &docs
+        name: Build Docs
+        caches:
+          - pip
+        script:
+          - ./ci_scripts/latest/execute_script.sh test_sphinx.sh
+    - step: &push
+        caches:
+          - pip
+        name: Push
+        script:
+          - ./ci_scripts/latest/execute_script.sh push_python.sh
+
+pipelines:
+  default:
+    - step: *download_execute_script
+    - parallel:
+      - step: *lint
+      - step: *changelog
+    - parallel:
+      - step: *scan
+      - step: *test
+      - step: *docs
+  tags:
+    'v*':
+      - step: *download_execute_script
+      - parallel:
+        - step: *changelog
+        - step: *lint
+      - parallel:
+        - step: *scan
+        - step: *test
+        - step: *docs
+      - step: *push

dkist_processing_math-2.2.0rc1/changelog/18.misc.rst

@@ -0,0 +1 @@
+Update bitbucket pipelines to use common scripts for checking for changelog snippets and verifying doc builds.

dkist_processing_math-2.2.0rc1/changelog/19.misc.rst

@@ -0,0 +1 @@
+Update Bitbucket pipelines to use execute script for standard steps.

dkist_processing_math-2.2.0rc1/changelog/20.feature.rst

@@ -0,0 +1,2 @@
+Add `stddev_numpy_arrays` function for computing the per-pixel standard deviation of an iterable of arrays.
+If a `generator` is provided this function is very memory-light, even for huge inputs.

dkist_processing_math-2.2.0rc1/dkist_processing_math/statistics.py

@@ -0,0 +1,102 @@
+"""Statistics."""
+from typing import Iterable
+from typing import Union
+
+import numpy as np
+
+
+def average_numpy_arrays(arrays: Union[Iterable[np.ndarray], np.ndarray]) -> np.ndarray:
+    """
+    Given an iterable of numpy arrays, calculate the pixel-wise average and return it in a numpy array.
+
+    This will work for a single array as well, just in case...
+
+    Parameters
+    ----------
+    arrays
+        The arrays to be averaged
+
+    Returns
+    -------
+    The average of the input arrays
+
+    """
+    arrays = [arrays] if isinstance(arrays, np.ndarray) else arrays
+    count = 0  # This statement is here only to suppress an uninitialized variable warning
+    output = None
+    for count, array in enumerate(arrays):
+        if output is None:
+            output = np.array(array).astype(float)
+        else:
+            if array.shape != output.shape:
+                raise ValueError(
+                    f"All arrays must be the same shape. "
+                    f"Shape of initial array = {output.shape} "
+                    f"Shape of current array = {array.shape}"
+                )
+            output += array
+    if output is not None:
+        return output / (count + 1)
+    raise ValueError("data_arrays is empty")
+
+
+def stddev_numpy_arrays(arrays: Iterable[np.ndarray]) -> np.ndarray:
+    r"""
+    Given an iterable of numpy arrays, calculate the pixel-wise standard deviation and return it in a numpy array.
+
+    This function uses Welford's single-pass algorithm. The standard deviation for the :math:`n^{th}` array is
+
+    .. math::
+        \sigma_n = \sqrt{\frac{M_{2,n}}{n}}
+
+    where
+
+    .. math::
+        M_{2,n} = M_{2,n-1} + (x_n - \bar{x}_{n-1})(x_n - \bar{x}_n)
+
+    and
+
+    .. math::
+        \bar{x}_n = \bar{x}_{n-1} + \frac{x_n - \bar{x}_{n-1}}{n}
+
+    Note that in computing :math:`\sigma` we divide by :math:`n` instead of :math:`(n-1)`. This is to match the default
+    behavior of `numpy.std`.
+
+    Parameters
+    ----------
+    arrays
+        The arrays to be to compute the standard deviation of
+
+    Returns
+    -------
+    The standard deviation of the input arrays
+    """
+    if isinstance(arrays, np.ndarray):
+        raise ValueError(
+            "Need an Iterator with at least two arrays, but a single array was provided."
+        )
+
+    running_mean = 0.0
+    running_M2 = 0.0
+    count = 0
+    for array in arrays:
+        count += 1
+        try:
+            delta = array - running_mean
+        except ValueError:
+            raise ValueError(
+                f"All arrays must be the same shape. "
+                f"Shape of initial array = {running_mean.shape} "
+                f"Shape of current array = {array.shape}"
+            )
+        running_mean += delta / count
+        delta2 = array - running_mean
+        running_M2 += delta * delta2
+
+    match count:
+        case 0:
+            raise ValueError("arrays is empty")
+        case 1:
+            raise ValueError(f"Need at least two arrays. Provided Iterator only has {count}")
+        case _:
+            return np.sqrt(running_M2 / count)

{dkist_processing_math-2.1.2 → dkist_processing_math-2.2.0rc1}/dkist_processing_math/tests/conftest.py

@@ -4,8 +4,13 @@ import pytest
 rng = np.random.default_rng()
 
 
+@pytest.fixture
+def test_array_shape() -> tuple[int, int]:
+    return (10, 10)
+
+
 @pytest.fixture()
-def numpy_arrays():
+def numpy_arrays(test_array_shape):
     """
     Create an iterable of random 10 x 10 arrays
 
@@ -14,7 +19,11 @@ def numpy_arrays():
     List[np.ndarray]
 
     """
-    arrays = [rng.standard_normal((10, 10)), rng.standard_normal((10, 10))]
+    arrays = [
+        rng.standard_normal(test_array_shape),
+        rng.standard_normal(test_array_shape),
+        rng.standard_normal(test_array_shape),
+    ]
     return arrays
 
 
@@ -32,7 +41,7 @@ def numpy_array(numpy_arrays):
 
 
 @pytest.fixture()
-def numpy_arrays_wrong_shape():
+def numpy_arrays_wrong_shape(test_array_shape):
     """
     Create an iterable of random arrays of differing sizes
 
@@ -41,7 +50,10 @@ def numpy_arrays_wrong_shape():
     List[np.ndarray]
 
     """
-    arrays = [rng.standard_normal((10, 10)), rng.standard_normal((10, 15))]
+    arrays = [
+        rng.standard_normal(test_array_shape),
+        rng.standard_normal(tuple(i * 2 for i in test_array_shape)),
+    ]
     return arrays
 
 

{dkist_processing_math-2.1.2 → dkist_processing_math-2.2.0rc1}/dkist_processing_math/tests/test_statistics.py

@@ -2,6 +2,7 @@ import numpy as np
 import pytest
 
 from dkist_processing_math.statistics import average_numpy_arrays
+from dkist_processing_math.statistics import stddev_numpy_arrays
 
 
 @pytest.fixture()
@@ -60,3 +61,24 @@ def test_average_with_large_values(int_arrays_with_large_values):
     """
     expected = np.ones((10, 10)) * 17500
     np.testing.assert_equal(expected, average_numpy_arrays(int_arrays_with_large_values))
+
+
+def test_stddev_numpy_arrays(numpy_arrays):
+    expected_result = np.std(numpy_arrays, axis=0)
+    result = stddev_numpy_arrays(numpy_arrays)
+
+    np.testing.assert_allclose(result, expected_result)
+
+
+def test_stddec_numpy_arrays_failures(numpy_arrays, numpy_arrays_wrong_shape):
+    with pytest.raises(ValueError, match="Need an Iterator with at least two arrays"):
+        stddev_numpy_arrays(numpy_arrays[0])
+
+    with pytest.raises(ValueError, match="All arrays must be the same shape"):
+        stddev_numpy_arrays(numpy_arrays_wrong_shape)
+
+    with pytest.raises(ValueError, match="Need at least two arrays"):
+        stddev_numpy_arrays([numpy_arrays[0]])
+
+    with pytest.raises(ValueError, match="arrays is empty"):
+        stddev_numpy_arrays([])

{dkist_processing_math-2.1.2 → dkist_processing_math-2.2.0rc1}/dkist_processing_math.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: dkist-processing-math
-Version: 2.1.2
+Version: 2.2.0rc1
 Summary: Common math functionality used in the DKIST calibrations suite
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_math-2.1.2 → dkist_processing_math-2.2.0rc1}/dkist_processing_math.egg-info/SOURCES.txt

@@ -5,9 +5,11 @@
 CHANGELOG.rst
 README.rst
 bitbucket-pipelines.yml
-check_changelog_updated.sh
 pyproject.toml
 changelog/.gitempty
+changelog/18.misc.rst
+changelog/19.misc.rst
+changelog/20.feature.rst
 dkist_processing_math/__init__.py
 dkist_processing_math/arithmetic.py
 dkist_processing_math/feature.py

{dkist_processing_math-2.1.2 → dkist_processing_math-2.2.0rc1}/dkist_processing_math.egg-info/top_level.txt

@@ -1,4 +1,5 @@
 changelog
+ci_scripts
 dkist_processing_math
 docs
 licenses

dkist_processing_math-2.1.2/bitbucket-pipelines.yml

@@ -1,100 +0,0 @@
-image: python:3.11
-definitions:
-  steps:
-    - step: &lint
-        caches:
-          - pip
-        name: Lint
-        script:
-          - echo "Retrieving and Executing Lint Script"
-          - curl -fL https://getcli.jfrog.io | sh
-          - ./jfrog rt dl --url $ARTIFACTORY_URL --user $ARTIFACTORY_USER --password $ARTIFACTORY_PASSWORD generic-packages/ci_scripts/latest/lint_python.sh
-          - cat ci_scripts/latest/lint_python.sh
-          - chmod 755 ci_scripts/latest/lint_python.sh
-          - ./ci_scripts/latest/lint_python.sh
-    - step: &changelog
-        caches:
-          - pip
-        name: Changelog
-        script:
-          - pip install towncrier
-          # We need to fetch the main branch to compare against
-          - git remote set-branches --add origin main
-          - git fetch origin main
-          - towncrier check --compare-with origin/main
-    - step: &scan
-        caches:
-          - pip
-        name: Scan
-        script:
-          - echo "Retrieving and Executing Scan Script"
-          - curl -fL https://getcli.jfrog.io | sh
-          - ./jfrog rt dl --url $ARTIFACTORY_URL --user $ARTIFACTORY_USER --password $ARTIFACTORY_PASSWORD generic-packages/ci_scripts/latest/scan_python.sh
-          - cat ci_scripts/latest/scan_python.sh
-          - chmod 755 ci_scripts/latest/scan_python.sh
-          - ./ci_scripts/latest/scan_python.sh
-    - step: &test
-        caches:
-          - pip
-        name: Test
-        script:
-          - pip install .[test]
-          - pytest --cov -m "not development" dkist_processing_math
-    - step: &docs
-        name: Build Docs
-        caches:
-          - pip
-        script:
-          - apt update && apt -y install graphviz
-          - pip install .[docs]
-          - sphinx-build --color -W --keep-going -b html docs docs/_build/html
-    - step: &check_changelog
-        name: Check for updated CHANGELOG
-        script:
-          - ./check_changelog_updated.sh
-    - step: &push
-        caches:
-          - pip
-        name: Push
-        script:
-          - pip install -U pip
-          - pip install build twine
-          - python -m build --outdir wheelhouse .
-          - python -m twine upload --skip-existing wheelhouse/*
-
-pipelines:
-  default:
-    - parallel:
-      - step: *lint
-      - step: *changelog
-    - parallel:
-      - step: *scan
-      - step: *test
-      - step: *docs
-  custom:
-    skip_doc_test_to_build_rc:  # I.e. skip the doc check (because we might have rc dependencies)
-      - variables:
-        - name: BITBUCKET_TAG
-          description: "Tag version to build (e.g., 'vX.Y.ZrcR')"
-      - step:
-          name: Validate Bitbucket pipeline input
-          script:
-            - git tag | grep -q ${BITBUCKET_TAG} || { echo Tag ${BITBUCKET_TAG} not found!; exit 1; }
-            - echo ${BITBUCKET_TAG} | grep -q "rc[0-9]\+$" || { echo Tag ${BITBUCKET_TAG} is not an rc. SHAME!; exit 1; }
-      - parallel:
-        - step: *lint
-        - step: *changelog
-      - parallel:
-        - step: *scan
-        - step: *test
-      - step: *push
-  tags:
-    'v*':
-      - parallel:
-        - step: *check_changelog
-        - step: *lint
-      - parallel:
-        - step: *scan
-        - step: *test
-        - step: *docs
-      - step: *push

dkist_processing_math-2.1.2/check_changelog_updated.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-CURRENT_VERSION=`git describe --tags --match v* --exact-match`
-ERROR_STR="CHANGELOG has not been updated for ${CURRENT_VERSION}\\nSee the README for instructions"
-ERROR_PAD="***********"
-
-if [[ $CURRENT_VERSION =~ "rc" ]]; then
-  echo "release candidate detected"
-  exit 0;
-fi
-
-ls changelog/*rst 2>/dev/null && {
-  echo -e "\\n${ERROR_PAD}"
-  echo -e "Unmerged changelog fragments found! You must run towncrier!\\nSee the README for instruction"
-  echo -e "${ERROR_PAD}"
-  exit 1;
-}
-
-head -n 1 CHANGELOG.rst | grep -q "^$CURRENT_VERSION (" - || {
-  echo -e "\\n${ERROR_PAD}"
-  echo -e $ERROR_STR;
-  echo -e "${ERROR_PAD}"
-  exit 1; }

dkist_processing_math-2.1.2/dkist_processing_math/statistics.py

@@ -1,40 +0,0 @@
-"""Statistics."""
-from typing import Iterable
-from typing import Union
-
-import numpy as np
-
-
-def average_numpy_arrays(arrays: Union[Iterable[np.ndarray], np.ndarray]) -> np.ndarray:
-    """
-    Given an iterable of numpy arrays, calculate the pixel-wise average and return it in a numpy array.
-
-    This will work for a single array as well, just in case...
-
-    Parameters
-    ----------
-    arrays
-        The arrays to be averaged
-
-    Returns
-    -------
-    The average of the input arrays
-
-    """
-    arrays = [arrays] if isinstance(arrays, np.ndarray) else arrays
-    count = 0  # This statement is here only to suppress an uninitialized variable warning
-    output = None
-    for count, array in enumerate(arrays):
-        if output is None:
-            output = np.array(array).astype(float)
-        else:
-            if array.shape != output.shape:
-                raise ValueError(
-                    f"All arrays must be the same shape. "
-                    f"Shape of initial array = {output.shape} "
-                    f"Shape of current array = {array.shape}"
-                )
-            output += array
-    if output is not None:
-        return output / (count + 1)
-    raise ValueError("data_arrays is empty")