psi-inference 0.1.0__tar.gz

psi_inference-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,20 @@
+name: ci
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install -e ".[dev]"
+      - run: ruff check .
+      - run: pytest -q

psi_inference-0.1.0/.gitignore

@@ -0,0 +1,8 @@
+.venv/
+__pycache__/
+*.pyc
+.pytest_cache/
+.ruff_cache/
+dist/
+build/
+*.egg-info/

psi_inference-0.1.0/CITATION.cff

@@ -0,0 +1,36 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+type: software
+title: "psi-inference: critical values and p-values for population stability testing"
+authors:
+  - family-names: Khobotov
+    given-names: Nikolai
+    orcid: "https://orcid.org/0009-0005-3612-830X"
+    email: nicholaask@gmail.com
+version: 0.1.0
+doi: 10.5281/zenodo.22342344
+identifiers:
+  - type: doi
+    value: 10.5281/zenodo.22342343
+    description: "Concept DOI for all versions; resolves to the latest."
+date-released: 2026-09-05
+url: "https://github.com/primaryaesthetics/psi-inference"
+repository-code: "https://github.com/primaryaesthetics/psi-inference"
+license: MIT
+keywords:
+  - population stability index
+  - PSI
+  - population resemblance statistic
+  - PRS
+  - credit risk
+  - model monitoring
+  - model validation
+abstract: >-
+  Critical values and p-values for population stability testing in credit
+  risk monitoring, in pure numpy and scipy. Implements the Yurdakul-Naranjo
+  null distribution of the population stability index, the Population
+  Resemblance Statistic of Potgieter, Van Zyl, Schutte and Lombard, and the
+  effect-size and overlapping tests of Du Pisanie, Allison, Budde and
+  Visagie. Every formula is pinned by a test to a worked example in its
+  source paper, and a Monte Carlo gate checks the analytic critical values
+  against the simulated null.

psi_inference-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Nikolai Khobotov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

psi_inference-0.1.0/PKG-INFO

@@ -0,0 +1,186 @@
+Metadata-Version: 2.5
+Name: psi-inference
+Version: 0.1.0
+Summary: Critical values and p-values for population stability testing: Yurdakul-Naranjo PSI, the Population Resemblance Statistic, and effect-size and overlapping tests
+Project-URL: Homepage, https://github.com/primaryaesthetics/psi-inference
+Project-URL: Repository, https://github.com/primaryaesthetics/psi-inference
+Project-URL: Issues, https://github.com/primaryaesthetics/psi-inference/issues
+Author-email: Nikolai Khobotov <nicholaask@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: PRS,PSI,credit risk,model monitoring,population stability index
+Classifier: Intended Audience :: Financial and Insurance Industry
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Mathematics
+Requires-Python: >=3.11
+Requires-Dist: numpy>=1.24
+Requires-Dist: scipy>=1.10
+Provides-Extra: dev
+Requires-Dist: feature-engine; extra == 'dev'
+Requires-Dist: iyipada; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# psi-inference
+
+[![PyPI](https://img.shields.io/pypi/v/psi-inference.svg)](https://pypi.org/project/psi-inference/)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22342343.svg)](https://doi.org/10.5281/zenodo.22342343)
+
+Critical values and p-values for population stability testing in credit risk
+monitoring: pure numpy and scipy, no pandas dependency, Python 3.11+.
+
+A model built on one population gets used on another one later, and the
+question a monitoring report has to answer is whether the two still look
+alike. The population stability index (PSI) is the standard tool for this,
+and the standard reading of it is a pair of constants: below 0.10 is fine,
+above 0.25 needs a rebuild. Those constants do not know the sample size or
+the number of bins the PSI was computed on, so the same reading means
+different things on a 500-row book and a 500,000-row one. This package
+implements the statistics that do know it.
+
+## Example
+
+```python
+import numpy as np
+from psi_inference import psi_test
+
+# Seven credit grades: the base sample and a later monitoring sample.
+base = np.array([253, 302, 204, 134, 72, 26, 8])      # 999 accounts
+current = np.array([177, 262, 285, 158, 88, 25, 6])    # 1001 accounts
+
+result = psi_test(base, current, alpha=0.05)
+print(result.statistic)        # 0.0677
+print(result.critical_value)   # 0.0252
+print(result.p_value)          # 7.2e-06
+print(result.reject)           # True
+```
+
+Read against the 0.10 rule of thumb, 0.068 says nothing has changed. This
+sample has seven categories and about a thousand rows on each side. At 5%
+significance the critical value here is 0.025. The observed PSI is more than
+two and a half times that. The p-value says the same thing in one number: a
+shift this large would happen by chance in roughly seven per hundred
+thousand comparisons if the population had not moved at all. The rule of
+thumb and the correctly sized test disagree here. On this sample size, the
+test is the one with a stated error rate.
+
+## Critical values
+
+At alpha = 0.05, from `psi_critical_value(n, m, bins, alpha=0.05)`:
+
+| n = m     | B = 10  | B = 20  |
+| --------- | ------- | ------- |
+| 1,000     | 0.0338  | 0.0603  |
+| 10,000    | 0.0034  | 0.0060  |
+| 100,000   | 0.0003  | 0.0006  |
+
+The critical value falls by roughly an order of magnitude for each order of
+magnitude of sample size, and rises with the number of bins. A PSI of 0.05
+is a strong signal at 100,000 rows per side and unremarkable at 1,000. Fixed
+thresholds cannot see either fact.
+
+## Why 0.10 and 0.25 are not thresholds
+
+The 0.10 / 0.25 pair comes from Lewis (1994) as a rule of thumb, with no
+stated error rate and no dependence on how much data the PSI was computed on.
+Yurdakul and Naranjo (2020) show that under the hypothesis of no population
+shift, PSI is approximately distributed as (1/n + 1/m) times a chi-square
+random variable with B − 1 degrees of freedom. Here n and m are the two
+sample sizes and B is the number of bins. That scale factor is exactly what
+the fixed thresholds throw away. The practical consequence is two-sided: a
+small monitoring sample crosses 0.10 routinely on pure sampling noise, and a
+large one can move by an amount that would fail any honest test while never
+approaching 0.25. Reporting a PSI against 0.10 or 0.25 is reporting a number
+against constants that were never fit to this sample size, this bin count,
+or any stated tolerance for being wrong.
+
+## What this package implements
+
+- **PSI** (`psi_statistic`, `psi_critical_value`, `psi_pvalue`, `psi_test`):
+  Yurdakul, B. and Naranjo, J. (2020), "Statistical properties of the
+  population stability index", *Journal of Risk Model Validation* 14(4),
+  89-100, section 3 (theorems 3.1-3.3) and section 4 (equation 4.1). Two
+  other Python packages compute the same asymptotic critical value, `iyipada`
+  and `feature-engine`'s `DropHighPSIFeatures(threshold="auto")`; this module
+  is checked against both in `tests/test_cross_check.py` and is included here
+  for completeness, not as a new result.
+- **PRS** (`prs_statistic`, `prs_delta`, `prs_thresholds`, `prs_classify`):
+  the Population Resemblance Statistic of Potgieter, C.J., Van Zyl, C.,
+  Schutte, W.D. and Lombard, F., "The Population Resemblance Statistic: A
+  Chi-Square Measure of Fit for Banking" (arXiv:2307.11878). Compares one
+  sample against a fixed reference population rather than two samples
+  against each other, and gives sample-size-calibrated critical values from
+  the non-central chi-square distribution, plus a three-tier decision
+  (acceptable / partial / full discrepancy) instead of a single cutoff.
+- **Effect-size test** (`effect_size_statistic`, `effect_size_critical_value`,
+  `effect_size_pvalue`) and **overlapping test** (`overlapping_statistic`,
+  `overlapping_critical_value`, `overlapping_pvalue`): Du Pisanie, J.,
+  Allison, J.S., Budde, C.J. and Visagie, I.J.H., "A critical review of
+  existing and new population stability testing procedures in credit risk
+  scoring" (arXiv:2303.01227), sections 3.6 and 3.7. The effect-size
+  statistic does not grow with sample size, which is the failure mode of the
+  classical goodness-of-fit tests on a bank-sized book. The overlapping
+  statistic reports the share of probability mass two distributions hold in
+  common, which needs no explanation of a p-value to a non-technical
+  audience.
+
+None of these three had a Python implementation before this package.
+`rpsi` on CRAN implements the Yurdakul-Naranjo critical value in R and
+nothing beyond it.
+
+## Installing and running the tests
+
+```
+pip install psi-inference
+```
+
+To run the tests from a checkout:
+
+```
+uv venv --python 3.12
+uv pip install -e ".[dev]"
+uv run pytest
+```
+
+The test suite includes a Monte Carlo gate. It simulates the null directly
+for several sample sizes and bin counts, including a small sample (500 rows)
+and unequal sample sizes (50,000 against 2,000). It checks that the
+empirical 95th percentile of the simulated statistic matches the analytic
+critical value within simulation error for the PSI and for the PRS. A
+formula recalled wrong from the paper produces a number, not an error. This
+is the check that catches it.
+
+## Notes on the sources
+
+Every formula in this package is pinned by a test to a worked example
+printed in its source paper: the PSI to Yurdakul and Naranjo's seven-grade
+example and to their critical-value tables for B = 10 and B = 20; the PRS to
+the four (sample size, bin count) cases of the source paper's critical-value
+table; the effect-size and overlapping statistics to the paper's example of
+a two-category shift from 50/50 to 50.5/49.5. Each matches to the digits the
+source prints, and the test names say which table.
+
+The PSI null agrees everywhere it was checked: Yurdakul and Naranjo's own
+equation, the `rpsi` R source, `iyipada` and `feature-engine` all reduce to
+the same line, `chi2.ppf(1 - alpha, bins - 1) * (1/n + 1/m)`. `rpsi` also
+offers a mode that treats the base distribution as a fixed population rather
+than a random sample, which drops the `1/n` term. That mode is not
+implemented here. The PRS covers the fixed-reference case with a null that
+is built for it.
+
+## Citing
+
+The package is archived on Zenodo. The concept DOI 10.5281/zenodo.22342343
+resolves to the latest version; the DOI of version 0.1.0 is
+10.5281/zenodo.22342344. `CITATION.cff` in the repository carries the full
+record.
+
+## Licence
+
+MIT.

psi_inference-0.1.0/README.md

@@ -0,0 +1,157 @@
+# psi-inference
+
+[![PyPI](https://img.shields.io/pypi/v/psi-inference.svg)](https://pypi.org/project/psi-inference/)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22342343.svg)](https://doi.org/10.5281/zenodo.22342343)
+
+Critical values and p-values for population stability testing in credit risk
+monitoring: pure numpy and scipy, no pandas dependency, Python 3.11+.
+
+A model built on one population gets used on another one later, and the
+question a monitoring report has to answer is whether the two still look
+alike. The population stability index (PSI) is the standard tool for this,
+and the standard reading of it is a pair of constants: below 0.10 is fine,
+above 0.25 needs a rebuild. Those constants do not know the sample size or
+the number of bins the PSI was computed on, so the same reading means
+different things on a 500-row book and a 500,000-row one. This package
+implements the statistics that do know it.
+
+## Example
+
+```python
+import numpy as np
+from psi_inference import psi_test
+
+# Seven credit grades: the base sample and a later monitoring sample.
+base = np.array([253, 302, 204, 134, 72, 26, 8])      # 999 accounts
+current = np.array([177, 262, 285, 158, 88, 25, 6])    # 1001 accounts
+
+result = psi_test(base, current, alpha=0.05)
+print(result.statistic)        # 0.0677
+print(result.critical_value)   # 0.0252
+print(result.p_value)          # 7.2e-06
+print(result.reject)           # True
+```
+
+Read against the 0.10 rule of thumb, 0.068 says nothing has changed. This
+sample has seven categories and about a thousand rows on each side. At 5%
+significance the critical value here is 0.025. The observed PSI is more than
+two and a half times that. The p-value says the same thing in one number: a
+shift this large would happen by chance in roughly seven per hundred
+thousand comparisons if the population had not moved at all. The rule of
+thumb and the correctly sized test disagree here. On this sample size, the
+test is the one with a stated error rate.
+
+## Critical values
+
+At alpha = 0.05, from `psi_critical_value(n, m, bins, alpha=0.05)`:
+
+| n = m     | B = 10  | B = 20  |
+| --------- | ------- | ------- |
+| 1,000     | 0.0338  | 0.0603  |
+| 10,000    | 0.0034  | 0.0060  |
+| 100,000   | 0.0003  | 0.0006  |
+
+The critical value falls by roughly an order of magnitude for each order of
+magnitude of sample size, and rises with the number of bins. A PSI of 0.05
+is a strong signal at 100,000 rows per side and unremarkable at 1,000. Fixed
+thresholds cannot see either fact.
+
+## Why 0.10 and 0.25 are not thresholds
+
+The 0.10 / 0.25 pair comes from Lewis (1994) as a rule of thumb, with no
+stated error rate and no dependence on how much data the PSI was computed on.
+Yurdakul and Naranjo (2020) show that under the hypothesis of no population
+shift, PSI is approximately distributed as (1/n + 1/m) times a chi-square
+random variable with B − 1 degrees of freedom. Here n and m are the two
+sample sizes and B is the number of bins. That scale factor is exactly what
+the fixed thresholds throw away. The practical consequence is two-sided: a
+small monitoring sample crosses 0.10 routinely on pure sampling noise, and a
+large one can move by an amount that would fail any honest test while never
+approaching 0.25. Reporting a PSI against 0.10 or 0.25 is reporting a number
+against constants that were never fit to this sample size, this bin count,
+or any stated tolerance for being wrong.
+
+## What this package implements
+
+- **PSI** (`psi_statistic`, `psi_critical_value`, `psi_pvalue`, `psi_test`):
+  Yurdakul, B. and Naranjo, J. (2020), "Statistical properties of the
+  population stability index", *Journal of Risk Model Validation* 14(4),
+  89-100, section 3 (theorems 3.1-3.3) and section 4 (equation 4.1). Two
+  other Python packages compute the same asymptotic critical value, `iyipada`
+  and `feature-engine`'s `DropHighPSIFeatures(threshold="auto")`; this module
+  is checked against both in `tests/test_cross_check.py` and is included here
+  for completeness, not as a new result.
+- **PRS** (`prs_statistic`, `prs_delta`, `prs_thresholds`, `prs_classify`):
+  the Population Resemblance Statistic of Potgieter, C.J., Van Zyl, C.,
+  Schutte, W.D. and Lombard, F., "The Population Resemblance Statistic: A
+  Chi-Square Measure of Fit for Banking" (arXiv:2307.11878). Compares one
+  sample against a fixed reference population rather than two samples
+  against each other, and gives sample-size-calibrated critical values from
+  the non-central chi-square distribution, plus a three-tier decision
+  (acceptable / partial / full discrepancy) instead of a single cutoff.
+- **Effect-size test** (`effect_size_statistic`, `effect_size_critical_value`,
+  `effect_size_pvalue`) and **overlapping test** (`overlapping_statistic`,
+  `overlapping_critical_value`, `overlapping_pvalue`): Du Pisanie, J.,
+  Allison, J.S., Budde, C.J. and Visagie, I.J.H., "A critical review of
+  existing and new population stability testing procedures in credit risk
+  scoring" (arXiv:2303.01227), sections 3.6 and 3.7. The effect-size
+  statistic does not grow with sample size, which is the failure mode of the
+  classical goodness-of-fit tests on a bank-sized book. The overlapping
+  statistic reports the share of probability mass two distributions hold in
+  common, which needs no explanation of a p-value to a non-technical
+  audience.
+
+None of these three had a Python implementation before this package.
+`rpsi` on CRAN implements the Yurdakul-Naranjo critical value in R and
+nothing beyond it.
+
+## Installing and running the tests
+
+```
+pip install psi-inference
+```
+
+To run the tests from a checkout:
+
+```
+uv venv --python 3.12
+uv pip install -e ".[dev]"
+uv run pytest
+```
+
+The test suite includes a Monte Carlo gate. It simulates the null directly
+for several sample sizes and bin counts, including a small sample (500 rows)
+and unequal sample sizes (50,000 against 2,000). It checks that the
+empirical 95th percentile of the simulated statistic matches the analytic
+critical value within simulation error for the PSI and for the PRS. A
+formula recalled wrong from the paper produces a number, not an error. This
+is the check that catches it.
+
+## Notes on the sources
+
+Every formula in this package is pinned by a test to a worked example
+printed in its source paper: the PSI to Yurdakul and Naranjo's seven-grade
+example and to their critical-value tables for B = 10 and B = 20; the PRS to
+the four (sample size, bin count) cases of the source paper's critical-value
+table; the effect-size and overlapping statistics to the paper's example of
+a two-category shift from 50/50 to 50.5/49.5. Each matches to the digits the
+source prints, and the test names say which table.
+
+The PSI null agrees everywhere it was checked: Yurdakul and Naranjo's own
+equation, the `rpsi` R source, `iyipada` and `feature-engine` all reduce to
+the same line, `chi2.ppf(1 - alpha, bins - 1) * (1/n + 1/m)`. `rpsi` also
+offers a mode that treats the base distribution as a fixed population rather
+than a random sample, which drops the `1/n` term. That mode is not
+implemented here. The PRS covers the fixed-reference case with a null that
+is built for it.
+
+## Citing
+
+The package is archived on Zenodo. The concept DOI 10.5281/zenodo.22342343
+resolves to the latest version; the DOI of version 0.1.0 is
+10.5281/zenodo.22342344. `CITATION.cff` in the repository carries the full
+record.
+
+## Licence
+
+MIT.

psi_inference-0.1.0/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "psi-inference"
+version = "0.1.0"
+description = "Critical values and p-values for population stability testing: Yurdakul-Naranjo PSI, the Population Resemblance Statistic, and effect-size and overlapping tests"
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+license-files = ["LICENSE"]
+authors = [{ name = "Nikolai Khobotov", email = "nicholaask@gmail.com" }]
+keywords = [
+    "population stability index",
+    "credit risk",
+    "model monitoring",
+    "PSI",
+    "PRS",
+]
+classifiers = [
+    "Intended Audience :: Financial and Insurance Industry",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Operating System :: OS Independent",
+    "Topic :: Scientific/Engineering :: Mathematics",
+]
+dependencies = [
+    "numpy>=1.24",
+    "scipy>=1.10",
+]
+
+[project.urls]
+Homepage = "https://github.com/primaryaesthetics/psi-inference"
+Repository = "https://github.com/primaryaesthetics/psi-inference"
+Issues = "https://github.com/primaryaesthetics/psi-inference/issues"
+
+[project.optional-dependencies]
+dev = ["pytest>=8", "ruff>=0.6", "iyipada", "feature-engine"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/psi_inference"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+
+[tool.ruff]
+line-length = 100

psi_inference-0.1.0/src/psi_inference/__init__.py

@@ -0,0 +1,75 @@
+"""Critical values and p-values for population stability testing.
+
+Four statistics used to test whether the population a model scores today
+still resembles the one it was built on, each with the inference the
+rule-of-thumb thresholds of 0.10 and 0.25 never had:
+
+- `psi_statistic`, `psi_critical_value`, `psi_pvalue`, `psi_test` --
+  the population stability index, tested against Yurdakul and Naranjo's
+  (2020) asymptotic chi-square null.
+- `prs_statistic`, `prs_delta`, `prs_thresholds`, `prs_classify` --
+  the Population Resemblance Statistic of Potgieter, Van Zyl, Schutte and
+  Lombard (arXiv:2307.11878), a fixed-reference alternative with
+  sample-size-calibrated critical values from the non-central chi-square.
+- `effect_size_statistic`, `effect_size_critical_value`, `effect_size_pvalue` --
+  the category-level effect-size test of Du Pisanie, Allison, Budde and
+  Visagie (arXiv:2303.01227), which does not grow with sample size.
+- `overlapping_statistic`, `overlapping_critical_value`, `overlapping_pvalue` --
+  the overlapping statistic from the same paper, the share of probability
+  mass two distributions hold in common.
+
+See the README for which statistic to reach for and why 0.10/0.25 are not
+thresholds.
+"""
+
+from .effect_size import (
+    effect_size_critical_value,
+    effect_size_per_category,
+    effect_size_pvalue,
+    effect_size_statistic,
+)
+from .overlapping import (
+    overlapping_critical_value,
+    overlapping_pvalue,
+    overlapping_statistic,
+)
+from .prs import (
+    PRSThresholds,
+    prs_classify,
+    prs_delta,
+    prs_noncentrality_sup,
+    prs_statistic,
+    prs_thresholds,
+)
+from .yurdakul import (
+    PSITestResult,
+    psi_critical_value,
+    psi_pvalue,
+    psi_scale,
+    psi_statistic,
+    psi_test,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "PRSThresholds",
+    "PSITestResult",
+    "effect_size_critical_value",
+    "effect_size_per_category",
+    "effect_size_pvalue",
+    "effect_size_statistic",
+    "overlapping_critical_value",
+    "overlapping_pvalue",
+    "overlapping_statistic",
+    "prs_classify",
+    "prs_delta",
+    "prs_noncentrality_sup",
+    "prs_statistic",
+    "prs_thresholds",
+    "psi_critical_value",
+    "psi_pvalue",
+    "psi_scale",
+    "psi_statistic",
+    "psi_test",
+]

psi_inference-0.1.0/src/psi_inference/bootstrap.py

@@ -0,0 +1,89 @@
+"""The parametric-bootstrap null shared by the effect-size and overlapping tests.
+
+Du Pisanie, J., Allison, J.S., Budde, C.J. and Visagie, I.J.H., "A critical
+review of existing and new population stability testing procedures in credit
+risk scoring" (arXiv:2303.01227), section 3.1.
+
+Unlike the PSI and the PRS, the effect-size statistic and the overlapping
+statistic have no closed-form asymptotic null in the paper. The paper's own
+answer, stated as a five-step algorithm in section 3.1, is to simulate it:
+since the reference distribution q is treated as fixed and known, a
+realisation of the current sample can be drawn directly from
+Multinomial(m, q), with no need for the observed sample at all.
+
+    1. Draw a realisation M* from Multinomial(m, q); let P* = M*/m.
+    2. Compute Delta* = Delta(q, P*).
+    3. Repeat steps 1-2 b times to get Delta*_1, ..., Delta*_b.
+    4. Sort them: Delta*_(1) <= ... <= Delta*_(b).
+    5. The critical value at level alpha is Delta*_(beta), beta = floor(b*(1-alpha)).
+
+and the p-value of an observed value is the share of the b simulated values
+at least as large as it. The paper notes this same algorithm covers every
+statistic considered except the overlapping measure, for which the required
+substitution is Delta(q, P) = 1 - eta(q, P) -- handled by `overlapping.py`,
+not by this module, which only ever sees an already-increasing-in-discrepancy
+statistic.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import numpy as np
+
+
+def bootstrap_null(
+    reference: np.ndarray,
+    m: int,
+    statistic: Callable[[np.ndarray, np.ndarray], float],
+    *,
+    draws: int = 10_000,
+    rng: np.random.Generator | int | None = None,
+) -> np.ndarray:
+    """b draws of `statistic(reference, P*)` under Multinomial(m, reference).
+
+    `statistic` must take (reference_distribution, sample_proportions) and
+    return a single float that increases with discrepancy from the reference.
+    """
+    reference = np.asarray(reference, dtype=np.float64)
+    if m < 1:
+        raise ValueError("m must be at least 1")
+    if draws < 1:
+        raise ValueError("draws must be at least 1")
+    if isinstance(rng, (int, np.integer)) or rng is None:
+        rng = np.random.default_rng(rng)
+    counts = rng.multinomial(m, reference, size=draws)
+    p_star = counts / m
+    return np.array([statistic(reference, row) for row in p_star])
+
+
+def bootstrap_critical_value(
+    reference: np.ndarray,
+    m: int,
+    statistic: Callable[[np.ndarray, np.ndarray], float],
+    alpha: float = 0.05,
+    *,
+    draws: int = 10_000,
+    rng: np.random.Generator | int | None = None,
+) -> float:
+    """The order statistic at floor(b*(1-alpha)) of the simulated null (step 5)."""
+    if not 0 < alpha < 1:
+        raise ValueError("alpha must be in (0, 1)")
+    null = np.sort(bootstrap_null(reference, m, statistic, draws=draws, rng=rng))
+    b = null.shape[0]
+    index = min(int(np.floor(b * (1.0 - alpha))), b - 1)
+    return float(null[index])
+
+
+def bootstrap_pvalue(
+    observed: float,
+    reference: np.ndarray,
+    m: int,
+    statistic: Callable[[np.ndarray, np.ndarray], float],
+    *,
+    draws: int = 10_000,
+    rng: np.random.Generator | int | None = None,
+) -> float:
+    """The share of the simulated null at least as large as the observed value."""
+    null = bootstrap_null(reference, m, statistic, draws=draws, rng=rng)
+    return float(np.mean(null >= observed))