validity-screen 0.3.0__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- validity_screen-0.4.1/PKG-INFO +120 -0
- validity_screen-0.4.1/README.md +89 -0
- {validity_screen-0.3.0 → validity_screen-0.4.1}/pyproject.toml +6 -2
- validity_screen-0.4.1/tests/test_core.py +401 -0
- {validity_screen-0.3.0 → validity_screen-0.4.1}/validity_screen/__init__.py +12 -7
- validity_screen-0.4.1/validity_screen/cli.py +189 -0
- validity_screen-0.4.1/validity_screen/core.py +636 -0
- validity_screen-0.4.1/validity_screen.egg-info/PKG-INFO +120 -0
- validity_screen-0.4.1/validity_screen.egg-info/requires.txt +6 -0
- validity_screen-0.3.0/PKG-INFO +0 -119
- validity_screen-0.3.0/README.md +0 -92
- validity_screen-0.3.0/tests/test_core.py +0 -94
- validity_screen-0.3.0/validity_screen/cli.py +0 -122
- validity_screen-0.3.0/validity_screen/core.py +0 -373
- validity_screen-0.3.0/validity_screen.egg-info/PKG-INFO +0 -119
- validity_screen-0.3.0/validity_screen.egg-info/requires.txt +0 -2
- {validity_screen-0.3.0 → validity_screen-0.4.1}/LICENSE +0 -0
- {validity_screen-0.3.0 → validity_screen-0.4.1}/setup.cfg +0 -0
- {validity_screen-0.3.0 → validity_screen-0.4.1}/validity_screen.egg-info/SOURCES.txt +0 -0
- {validity_screen-0.3.0 → validity_screen-0.4.1}/validity_screen.egg-info/dependency_links.txt +0 -0
- {validity_screen-0.3.0 → validity_screen-0.4.1}/validity_screen.egg-info/entry_points.txt +0 -0
- {validity_screen-0.3.0 → validity_screen-0.4.1}/validity_screen.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: validity-screen
|
|
3
|
+
Version: 0.4.1
|
|
4
|
+
Summary: Validity screen for LLM confidence signals: response sets, item sensitivity and planning
|
|
5
|
+
Author: Jon-Paul Cacioli
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/synthiumjp/validity-scaling-llm
|
|
8
|
+
Project-URL: Documentation, https://github.com/synthiumjp/validity-scaling-llm/tree/master/screen
|
|
9
|
+
Project-URL: Repository, https://github.com/synthiumjp/validity-scaling-llm
|
|
10
|
+
Project-URL: Issues, https://github.com/synthiumjp/validity-scaling-llm/issues
|
|
11
|
+
Keywords: llm,confidence,validity,metacognition,calibration,selective-prediction,screening,psychometrics,evaluation
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Requires-Python: >=3.9
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: numpy>=1.20
|
|
26
|
+
Requires-Dist: scipy>=1.7
|
|
27
|
+
Provides-Extra: test
|
|
28
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
29
|
+
Requires-Dist: scikit-learn>=1.0; extra == "test"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# validity-screen
|
|
33
|
+
|
|
34
|
+
**Check whether an LLM's confidence signal carries information about correctness before you build anything on it.**
|
|
35
|
+
|
|
36
|
+
[](https://pypi.org/project/validity-screen/)
|
|
37
|
+
[](https://opensource.org/licenses/MIT)
|
|
38
|
+
|
|
39
|
+
Abstention, routing and escalation all assume that a model's confidence varies with whether it is right. This package checks that assumption, using the logic psychological assessment applies to response validity: exclude responses the respondent did not give, rule out fixed response sets, and require evidence that responding follows content.
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install validity-screen
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Quick start
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
from validity_screen import screen
|
|
51
|
+
|
|
52
|
+
# Binary confidence (1 = kept / high, 0 = withdrawn / low)
|
|
53
|
+
r = screen(correct, keep, answered=answered, model_name="Model A")
|
|
54
|
+
|
|
55
|
+
# Continuous confidence: state the cut point you will deploy at, on the same scale as the data
|
|
56
|
+
r = screen(correct, confidence_0_100, answered=answered, cut=90)
|
|
57
|
+
|
|
58
|
+
print(r.tier, r.reason_code, r.flags) # e.g. Valid signal_detected []
|
|
59
|
+
print(r.vrs_table()) # full validity report
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`answered` marks rows to drop before scoring: declines, unparseable output, and values the pipeline supplied rather than the model (such as a parser default). Missing values on scored rows raise an error rather than being guessed; with `cut`, missing confidence values are excluded and counted.
|
|
63
|
+
|
|
64
|
+
## What it reports
|
|
65
|
+
|
|
66
|
+
From the correct/wrong × kept/withdrawn table:
|
|
67
|
+
|
|
68
|
+
| Index | Formula | Meaning |
|
|
69
|
+
|---|---|---|
|
|
70
|
+
| Kept-error rate `L` | b / (b + d) | share of errors on which confidence was kept |
|
|
71
|
+
| Withdrawn-correct rate `Fp` | c / (a + c) | share of correct answers withdrawn |
|
|
72
|
+
| Item sensitivity `J` | (1 − Fp) − L | Youden's J; binary AUROC = (1 + J) / 2 |
|
|
73
|
+
|
|
74
|
+
## Tiers (version 0.4)
|
|
75
|
+
|
|
76
|
+
| Tier | Rule |
|
|
77
|
+
|---|---|
|
|
78
|
+
| **Invalid** | blanket confidence (L ≥ .95) or blanket withdrawal (Fp ≥ .95), each certified by a Wilson lower bound > .90; or inversion (J < 0, one-sided mid-p < .025) |
|
|
79
|
+
| **Uncertifiable** | fewer than 10 errors or 10 correct answers, and no certified response set |
|
|
80
|
+
| **Indeterminate** | response set or inversion suspected but not certified, or no detectable sensitivity (`reason_code` says which) |
|
|
81
|
+
| **Valid** | J > 0 (one-sided Fisher mid-p < .025) and no response set |
|
|
82
|
+
|
|
83
|
+
Flags, reported beside the tier and never changing it: `low_coverage` (Fp ≥ .50), `response_set_not_excluded` (Valid, but the L or Fp 95% interval reaches .95), `fixed_responding` (one response on ≥ 95% of items).
|
|
84
|
+
|
|
85
|
+
**Valid means better than chance, not good.** Read J and its interval. With many items a very small J is Valid; use `min_j=` (chosen in advance) if you need a minimum effect.
|
|
86
|
+
|
|
87
|
+
## Operating characteristics
|
|
88
|
+
|
|
89
|
+
- **False Valid** (no real signal): at most .026 against a nominal .025 in typical cases (226 to 1,500 items, accuracy .70 to .97, keep rates .50 to .99). With only 10 to 12 errors it reaches about .030 at keep rates of .50 or more. At low keep rates (.10 to .30, as at very high cut points) it reaches about .037.
|
|
90
|
+
- **False Invalid**: a model whose true kept-error rate is .93 is certified as blanket confidence up to about 23% of the time, depending on the number of errors. The guard is set at a Wilson lower bound of .90.
|
|
91
|
+
- **Power** depends on the number of errors, not items. Use the planning functions:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
from validity_screen import power_valid, power_invalid, items_needed, errors_needed
|
|
95
|
+
power_valid(n_items=1000, accuracy=0.90, keep_correct=0.95, keep_wrong=0.85) # P(Valid) for J = .10
|
|
96
|
+
items_needed(accuracy=0.90, true_L=0.97, power=0.90) # 1600
|
|
97
|
+
errors_needed(withdrawn=0) # 35
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## Command line
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
validity-screen run --data results.csv --correct-col is_correct --confidence-col keep
|
|
104
|
+
validity-screen run --data results.csv --confidence-col confidence --cut 90
|
|
105
|
+
validity-screen plan --accuracy 0.92 --true-L 0.97
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Cells are read strictly. Blank or unreadable cells, including words in a numeric column, are excluded and counted, never guessed.
|
|
109
|
+
|
|
110
|
+
## Changes
|
|
111
|
+
|
|
112
|
+
See [CHANGELOG.md](CHANGELOG.md). Results from 0.1 should be rerun. Version 0.4 changed the decision rules; the decision record, with the rationale for each rule, is in the repository.
|
|
113
|
+
|
|
114
|
+
## Citation
|
|
115
|
+
|
|
116
|
+
Cacioli, J. P. (2026). *Screen Before You Interpret: A Validity Screen for LLM Confidence Signals and Its Operating Characteristics.* Preprint: arXiv:2604.17714.
|
|
117
|
+
|
|
118
|
+
## License
|
|
119
|
+
|
|
120
|
+
MIT
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# validity-screen
|
|
2
|
+
|
|
3
|
+
**Check whether an LLM's confidence signal carries information about correctness before you build anything on it.**
|
|
4
|
+
|
|
5
|
+
[](https://pypi.org/project/validity-screen/)
|
|
6
|
+
[](https://opensource.org/licenses/MIT)
|
|
7
|
+
|
|
8
|
+
Abstention, routing and escalation all assume that a model's confidence varies with whether it is right. This package checks that assumption, using the logic psychological assessment applies to response validity: exclude responses the respondent did not give, rule out fixed response sets, and require evidence that responding follows content.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install validity-screen
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Quick start
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
from validity_screen import screen
|
|
20
|
+
|
|
21
|
+
# Binary confidence (1 = kept / high, 0 = withdrawn / low)
|
|
22
|
+
r = screen(correct, keep, answered=answered, model_name="Model A")
|
|
23
|
+
|
|
24
|
+
# Continuous confidence: state the cut point you will deploy at, on the same scale as the data
|
|
25
|
+
r = screen(correct, confidence_0_100, answered=answered, cut=90)
|
|
26
|
+
|
|
27
|
+
print(r.tier, r.reason_code, r.flags) # e.g. Valid signal_detected []
|
|
28
|
+
print(r.vrs_table()) # full validity report
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
`answered` marks rows to drop before scoring: declines, unparseable output, and values the pipeline supplied rather than the model (such as a parser default). Missing values on scored rows raise an error rather than being guessed; with `cut`, missing confidence values are excluded and counted.
|
|
32
|
+
|
|
33
|
+
## What it reports
|
|
34
|
+
|
|
35
|
+
From the correct/wrong × kept/withdrawn table:
|
|
36
|
+
|
|
37
|
+
| Index | Formula | Meaning |
|
|
38
|
+
|---|---|---|
|
|
39
|
+
| Kept-error rate `L` | b / (b + d) | share of errors on which confidence was kept |
|
|
40
|
+
| Withdrawn-correct rate `Fp` | c / (a + c) | share of correct answers withdrawn |
|
|
41
|
+
| Item sensitivity `J` | (1 − Fp) − L | Youden's J; binary AUROC = (1 + J) / 2 |
|
|
42
|
+
|
|
43
|
+
## Tiers (version 0.4)
|
|
44
|
+
|
|
45
|
+
| Tier | Rule |
|
|
46
|
+
|---|---|
|
|
47
|
+
| **Invalid** | blanket confidence (L ≥ .95) or blanket withdrawal (Fp ≥ .95), each certified by a Wilson lower bound > .90; or inversion (J < 0, one-sided mid-p < .025) |
|
|
48
|
+
| **Uncertifiable** | fewer than 10 errors or 10 correct answers, and no certified response set |
|
|
49
|
+
| **Indeterminate** | response set or inversion suspected but not certified, or no detectable sensitivity (`reason_code` says which) |
|
|
50
|
+
| **Valid** | J > 0 (one-sided Fisher mid-p < .025) and no response set |
|
|
51
|
+
|
|
52
|
+
Flags, reported beside the tier and never changing it: `low_coverage` (Fp ≥ .50), `response_set_not_excluded` (Valid, but the L or Fp 95% interval reaches .95), `fixed_responding` (one response on ≥ 95% of items).
|
|
53
|
+
|
|
54
|
+
**Valid means better than chance, not good.** Read J and its interval. With many items a very small J is Valid; use `min_j=` (chosen in advance) if you need a minimum effect.
|
|
55
|
+
|
|
56
|
+
## Operating characteristics
|
|
57
|
+
|
|
58
|
+
- **False Valid** (no real signal): at most .026 against a nominal .025 in typical cases (226 to 1,500 items, accuracy .70 to .97, keep rates .50 to .99). With only 10 to 12 errors it reaches about .030 at keep rates of .50 or more. At low keep rates (.10 to .30, as at very high cut points) it reaches about .037.
|
|
59
|
+
- **False Invalid**: a model whose true kept-error rate is .93 is certified as blanket confidence up to about 23% of the time, depending on the number of errors. The guard is set at a Wilson lower bound of .90.
|
|
60
|
+
- **Power** depends on the number of errors, not items. Use the planning functions:
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from validity_screen import power_valid, power_invalid, items_needed, errors_needed
|
|
64
|
+
power_valid(n_items=1000, accuracy=0.90, keep_correct=0.95, keep_wrong=0.85) # P(Valid) for J = .10
|
|
65
|
+
items_needed(accuracy=0.90, true_L=0.97, power=0.90) # 1600
|
|
66
|
+
errors_needed(withdrawn=0) # 35
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Command line
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
validity-screen run --data results.csv --correct-col is_correct --confidence-col keep
|
|
73
|
+
validity-screen run --data results.csv --confidence-col confidence --cut 90
|
|
74
|
+
validity-screen plan --accuracy 0.92 --true-L 0.97
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Cells are read strictly. Blank or unreadable cells, including words in a numeric column, are excluded and counted, never guessed.
|
|
78
|
+
|
|
79
|
+
## Changes
|
|
80
|
+
|
|
81
|
+
See [CHANGELOG.md](CHANGELOG.md). Results from 0.1 should be rerun. Version 0.4 changed the decision rules; the decision record, with the rationale for each rule, is in the repository.
|
|
82
|
+
|
|
83
|
+
## Citation
|
|
84
|
+
|
|
85
|
+
Cacioli, J. P. (2026). *Screen Before You Interpret: A Validity Screen for LLM Confidence Signals and Its Operating Characteristics.* Preprint: arXiv:2604.17714.
|
|
86
|
+
|
|
87
|
+
## License
|
|
88
|
+
|
|
89
|
+
MIT
|
|
@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "validity-screen"
|
|
7
|
-
version = "0.
|
|
8
|
-
description = "Validity screen for LLM confidence signals
|
|
7
|
+
version = "0.4.1"
|
|
8
|
+
description = "Validity screen for LLM confidence signals: response sets, item sensitivity and planning"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {text = "MIT"}
|
|
11
11
|
requires-python = ">=3.9"
|
|
@@ -26,6 +26,7 @@ classifiers = [
|
|
|
26
26
|
"Programming Language :: Python :: 3.10",
|
|
27
27
|
"Programming Language :: Python :: 3.11",
|
|
28
28
|
"Programming Language :: Python :: 3.12",
|
|
29
|
+
"Programming Language :: Python :: 3.13",
|
|
29
30
|
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
30
31
|
]
|
|
31
32
|
dependencies = [
|
|
@@ -33,6 +34,9 @@ dependencies = [
|
|
|
33
34
|
"scipy>=1.7",
|
|
34
35
|
]
|
|
35
36
|
|
|
37
|
+
[project.optional-dependencies]
|
|
38
|
+
test = ["pytest>=7", "scikit-learn>=1.0"]
|
|
39
|
+
|
|
36
40
|
[project.urls]
|
|
37
41
|
Homepage = "https://github.com/synthiumjp/validity-scaling-llm"
|
|
38
42
|
Documentation = "https://github.com/synthiumjp/validity-scaling-llm/tree/master/screen"
|
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from validity_screen import (screen, binarise, errors_needed, items_needed, power_invalid, power_valid,
|
|
7
|
+
screen_batch, summary_table, wilson_ci, newcombe_diff_ci, midp_signal, MIN_N)
|
|
8
|
+
from validity_screen.cli import main as cli_main
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def make(nc, ni, keep_c, keep_i):
|
|
12
|
+
"""nc correct (keep_c kept), ni wrong (keep_i kept)."""
|
|
13
|
+
correct = np.r_[np.ones(nc, int), np.zeros(ni, int)]
|
|
14
|
+
keep = np.r_[np.ones(keep_c), np.zeros(nc - keep_c), np.ones(keep_i), np.zeros(ni - keep_i)].astype(int)
|
|
15
|
+
return correct, keep
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# ---------------- tiers (D1-D4) ----------------
|
|
19
|
+
|
|
20
|
+
def test_blanket_confidence_invalid():
|
|
21
|
+
r = screen(*make(460, 40, 460, 40))
|
|
22
|
+
assert r.tier == "Invalid" and r.reason_code == "blanket_confidence"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_blanket_withdrawal_invalid():
|
|
26
|
+
r = screen(*make(460, 40, 0, 0))
|
|
27
|
+
assert r.tier == "Invalid" and r.reason_code == "blanket_withdrawal"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_inversion_invalid():
|
|
31
|
+
r = screen(*make(460, 40, 92, 32)) # keep 20% of correct, 80% of errors
|
|
32
|
+
assert r.tier == "Invalid" and r.reason_code == "inversion"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_good_monitor_valid():
|
|
36
|
+
r = screen(*make(460, 40, 368, 16)) # keep 80% of correct, 40% of errors
|
|
37
|
+
assert r.tier == "Valid" and r.reason_code == "signal_detected" and r.flags == []
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_conservative_but_informative_is_valid_with_low_coverage():
|
|
41
|
+
# Fp = .55, L = 0: strong discrimination at a conservative operating point (v0.3 called this Invalid)
|
|
42
|
+
r = screen(*make(400, 100, 180, 0))
|
|
43
|
+
assert r.tier == "Valid" and "low_coverage" in r.flags
|
|
44
|
+
assert abs(r.J.value - 0.45) < 1e-12
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def test_uninformative_indeterminate():
|
|
48
|
+
r = screen(*make(466, 58, 373, 46)) # keep 80% of both
|
|
49
|
+
assert r.tier == "Indeterminate" and r.reason_code == "no_detectable_sensitivity"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_inversion_suspected():
|
|
53
|
+
r = screen(*make(466, 58, 373, 48)) # keeps errors slightly more, not significant
|
|
54
|
+
assert r.tier == "Indeterminate" and r.reason_code == "inversion_suspected"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_high_L_few_errors_suspected():
|
|
58
|
+
r = screen(*make(476, 24, 457, 23)) # L = .958 on 24 errors
|
|
59
|
+
assert r.tier == "Indeterminate" and r.reason_code == "blanket_confidence_suspected"
|
|
60
|
+
assert r.L.flag == "suspected"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_high_Fp_few_correct_suspected():
|
|
64
|
+
r = screen(*make(24, 476, 1, 10)) # Fp = .958 on 24 correct; not inverted
|
|
65
|
+
assert r.tier == "Indeterminate" and r.reason_code == "blanket_withdrawal_suspected"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_r1_like_is_uncertifiable_with_low_coverage():
|
|
69
|
+
r = screen(*make(446, 6, 24, 0)) # 6 errors, Fp = .946 (below .95)
|
|
70
|
+
assert r.tier == "Uncertifiable" and r.reason_code == "too_few_errors"
|
|
71
|
+
assert "low_coverage" in r.flags
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def test_blanket_withdrawal_certified_even_with_few_errors():
|
|
75
|
+
r = screen(*make(446, 6, 10, 0)) # Fp = .978, LB > .90, only 6 errors
|
|
76
|
+
assert r.tier == "Invalid" and r.reason_code == "blanket_withdrawal"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_too_few_correct():
|
|
80
|
+
r = screen(*make(8, 300, 4, 100))
|
|
81
|
+
assert r.tier == "Uncertifiable" and r.reason_code == "too_few_correct"
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_response_set_not_excluded_flag():
|
|
85
|
+
# Valid, but the L interval reaches .95 (few errors, high L)
|
|
86
|
+
r = screen(*make(470, 30, 420, 26))
|
|
87
|
+
assert r.tier == "Valid" or r.tier == "Indeterminate"
|
|
88
|
+
if r.tier == "Valid":
|
|
89
|
+
assert ("response_set_not_excluded" in r.flags) == (r.L.hi >= 0.95 or r.Fp.hi >= 0.95)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
# ---------------- boundaries ----------------
|
|
93
|
+
|
|
94
|
+
@pytest.mark.parametrize("ni,expect_estimable", [(MIN_N - 1, False), (MIN_N, True)])
|
|
95
|
+
def test_min_n_boundary_errors(ni, expect_estimable):
|
|
96
|
+
r = screen(*make(400, ni, 300, ni // 2))
|
|
97
|
+
assert (r.J.flag != "not_estimable") == expect_estimable
|
|
98
|
+
if not expect_estimable:
|
|
99
|
+
assert r.tier == "Uncertifiable"
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def test_response_set_threshold_boundary():
|
|
103
|
+
# exactly 95% kept on 140 errors with 7 withdrawn: certifiable (errors_needed(7) == 140)
|
|
104
|
+
r = screen(*make(400, 140, 380, 133))
|
|
105
|
+
assert r.L.value == pytest.approx(0.95) and r.tier == "Invalid"
|
|
106
|
+
# 139 errors with 7 withdrawn: L = .9496 < .95 -> not a response set
|
|
107
|
+
r = screen(*make(400, 139, 380, 132))
|
|
108
|
+
assert r.L.value < 0.95 and r.L.flag == "ok"
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def test_zero_cells():
|
|
112
|
+
for args in [(100, 20, 100, 0), (100, 20, 0, 20), (100, 20, 50, 0), (100, 20, 100, 20)]:
|
|
113
|
+
r = screen(*make(*args))
|
|
114
|
+
assert r.tier in {"Valid", "Invalid", "Indeterminate", "Uncertifiable"}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_empty_input():
|
|
118
|
+
r = screen([], [])
|
|
119
|
+
assert r.tier == "Uncertifiable" and r.n_scored == 0
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# ---------------- numerics ----------------
|
|
123
|
+
|
|
124
|
+
def test_newcombe_reference_value():
|
|
125
|
+
# Newcombe (1998), example 56/70 vs 48/80: interval .0524 to .3339
|
|
126
|
+
lo, hi = newcombe_diff_ci(56, 70, 48, 80)
|
|
127
|
+
assert lo == pytest.approx(0.0524, abs=1e-4) and hi == pytest.approx(0.3339, abs=1e-4)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def test_wilson_reference_value():
|
|
131
|
+
lo, hi = wilson_ci(81, 263) # Wilson interval for 81/263: .2553 to .3662
|
|
132
|
+
assert lo == pytest.approx(0.2553, abs=1e-4) and hi == pytest.approx(0.3662, abs=1e-4)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def test_midp_symmetry_and_range():
|
|
136
|
+
ps, pi = midp_signal(30, 10, 10, 30)
|
|
137
|
+
assert 0 <= ps <= 1 and 0 <= pi <= 1 and ps + pi == pytest.approx(1.0)
|
|
138
|
+
assert ps < 0.001
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def test_identities():
|
|
142
|
+
sk = pytest.importorskip("sklearn.metrics")
|
|
143
|
+
rng = np.random.default_rng(0)
|
|
144
|
+
for _ in range(100):
|
|
145
|
+
cor = (rng.random(400) < rng.uniform(.6, .95)).astype(int)
|
|
146
|
+
conf = np.where(rng.random(400) < .7, cor, 1 - cor)
|
|
147
|
+
conf = np.where(rng.random(400) < .5, 1, conf)
|
|
148
|
+
r = screen(cor, conf)
|
|
149
|
+
if np.isnan(r.J.value):
|
|
150
|
+
continue
|
|
151
|
+
assert abs(r.auroc_binary - sk.roc_auc_score(cor, conf)) < 1e-12
|
|
152
|
+
assert abs(r.youden_j - ((1 - r.Fp.value) - r.L.value)) < 1e-12
|
|
153
|
+
assert abs(r.rbs.value + r.J.value) < 1e-12
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
# ---------------- input checks (D8) ----------------
|
|
157
|
+
|
|
158
|
+
def test_rejects_raw_continuous_confidence():
|
|
159
|
+
c = np.r_[np.ones(450), np.zeros(50)]
|
|
160
|
+
conf = np.full(500, 90.0)
|
|
161
|
+
with pytest.raises(ValueError, match="cut="):
|
|
162
|
+
screen(c, conf)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def test_rejects_nan_confidence_on_scored_rows():
|
|
166
|
+
c, k = make(450, 50, 400, 20)
|
|
167
|
+
k = k.astype(float); k[3] = np.nan
|
|
168
|
+
with pytest.raises(ValueError, match="missing"):
|
|
169
|
+
screen(c, k)
|
|
170
|
+
ans = np.ones(500, bool); ans[3] = False
|
|
171
|
+
r = screen(c, k, answered=ans)
|
|
172
|
+
assert r.n_scored == 499
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def test_rejects_nan_correct():
|
|
176
|
+
c, k = make(450, 50, 400, 20)
|
|
177
|
+
c = c.astype(float); c[0] = np.nan
|
|
178
|
+
with pytest.raises(ValueError, match="missing"):
|
|
179
|
+
screen(c, k)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def test_cut_excludes_and_counts_missing():
|
|
183
|
+
c = np.r_[np.ones(450), np.zeros(50)]
|
|
184
|
+
conf = np.r_[np.full(450, 95.0), np.full(50, 60.0)]
|
|
185
|
+
conf[:5] = np.nan
|
|
186
|
+
r = screen(c, conf, cut=90)
|
|
187
|
+
assert r.n_missing == 5 and r.n_scored == 495 and r.binarisation_threshold == ">= 90"
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def test_cut_on_binary_rejected():
|
|
191
|
+
c, k = make(450, 50, 400, 20)
|
|
192
|
+
with pytest.raises(ValueError, match="looks binary"):
|
|
193
|
+
screen(c, k, cut=90)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def test_binarise_requires_threshold_and_rejects_nan():
|
|
197
|
+
x = np.array([10, 50, 90, 100])
|
|
198
|
+
assert binarise(x, 90).tolist() == [False, False, True, True]
|
|
199
|
+
with pytest.raises(TypeError):
|
|
200
|
+
binarise(x)
|
|
201
|
+
with pytest.raises(ValueError):
|
|
202
|
+
binarise(np.array([10, np.nan]), 50)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def test_exclusions():
|
|
206
|
+
c, k = make(460, 40, 368, 16)
|
|
207
|
+
ans = np.ones(500, int); ans[:10] = 0
|
|
208
|
+
r = screen(c, k, answered=ans)
|
|
209
|
+
assert r.n_excluded == 10 and r.n_scored == 490
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
# ---------------- planning ----------------
|
|
213
|
+
|
|
214
|
+
def test_errors_needed_integer_table():
|
|
215
|
+
assert [errors_needed(k) for k in range(8)] == [35, 53, 69, 84, 99, 113, 127, 140]
|
|
216
|
+
with pytest.raises(TypeError):
|
|
217
|
+
errors_needed(0.97)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def test_power_invalid_values():
|
|
221
|
+
assert power_invalid(1.0, 35) == pytest.approx(1.0)
|
|
222
|
+
assert power_invalid(0.97, 71) == pytest.approx(0.641, abs=0.001)
|
|
223
|
+
assert max(power_invalid(0.95, n) for n in (200, 500, 1000)) < 0.6
|
|
224
|
+
assert items_needed(0.92, 1.0) == 438
|
|
225
|
+
assert items_needed(0.92, 0.95) is None
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def test_false_valid_rate_held():
|
|
229
|
+
for acc, keep in [(.92, .95), (.95, .90), (.97, .99)]:
|
|
230
|
+
assert power_valid(452, acc, keep, keep) <= 0.025
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
# ---------------- batch, report, CLI ----------------
|
|
234
|
+
|
|
235
|
+
def test_batch_summary_report():
|
|
236
|
+
c, k = make(460, 40, 368, 16)
|
|
237
|
+
res = screen_batch({"a": {"correct": c, "confidence": k}, "b": {"correct": c, "confidence": np.ones(500)}})
|
|
238
|
+
s = summary_table(res)
|
|
239
|
+
assert "Invalid" in s and "Valid" in s
|
|
240
|
+
assert "VALIDITY REPORT" in res[0].vrs_table()
|
|
241
|
+
json.dumps(res[0].to_dict())
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _csv(tmp_path, rows, name="d.csv"):
|
|
245
|
+
p = tmp_path / name
|
|
246
|
+
p.write_text("correct,confidence\n" + "\n".join(f"{a},{b}" for a, b in rows) + "\n")
|
|
247
|
+
return str(p)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def test_cli_continuous_with_blank_cell(tmp_path, capsys):
|
|
251
|
+
rows = [(1, 95)] * 400 + [(1, 60)] * 50 + [(0, 95)] * 10 + [(0, 50)] * 40
|
|
252
|
+
rows[0] = (1, "") # one blank confidence cell
|
|
253
|
+
p = _csv(tmp_path, rows)
|
|
254
|
+
assert cli_main(["run", "--data", p, "--cut", "90", "--json"]) == 0
|
|
255
|
+
out = json.loads(capsys.readouterr().out)
|
|
256
|
+
assert out["n_missing"] == 1 and out["n_scored"] == 499 and out["tier"] == "Valid"
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def test_cli_continuous_requires_cut(tmp_path, capsys):
|
|
260
|
+
p = _csv(tmp_path, [(1, 95), (0, 40)] * 50)
|
|
261
|
+
with pytest.raises(SystemExit):
|
|
262
|
+
cli_main(["run", "--data", p])
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def test_cli_binary_words(tmp_path, capsys):
|
|
266
|
+
rows = [("True", "KEEP")] * 400 + [("True", "WITHDRAW")] * 50 + [("False", "KEEP")] * 10 + \
|
|
267
|
+
[("False", "WITHDRAW")] * 40
|
|
268
|
+
p = _csv(tmp_path, rows)
|
|
269
|
+
assert cli_main(["run", "--data", p, "--json"]) == 0
|
|
270
|
+
out = json.loads(capsys.readouterr().out)
|
|
271
|
+
assert out["tier"] == "Valid" and out["a"] == 400 and out["d"] == 40
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def test_cli_unreadable_cell_excluded(tmp_path, capsys):
|
|
275
|
+
rows = [("True", "KEEP")] * 400 + [("True", "WITHDRAW")] * 50 + [("False", "KEEP")] * 10 + \
|
|
276
|
+
[("False", "WITHDRAW")] * 40
|
|
277
|
+
rows[5] = ("True", "maybe")
|
|
278
|
+
p = _csv(tmp_path, rows)
|
|
279
|
+
assert cli_main(["run", "--data", p, "--json"]) == 0
|
|
280
|
+
out = json.loads(capsys.readouterr().out)
|
|
281
|
+
assert out["n_missing"] == 1 and out["n_scored"] == 499
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def test_cli_plan(capsys):
|
|
285
|
+
assert cli_main(["plan", "--accuracy", "0.92", "--true-L", "0.95"]) == 0
|
|
286
|
+
assert "cannot be reached" in capsys.readouterr().out
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
# ---------------- 0.4.1 regressions (independent review, defects P1-P13) ----------------
|
|
290
|
+
|
|
291
|
+
def _cont(n_c=450, n_w=50, hi=95.0, lo=60.0, scale=1.0):
|
|
292
|
+
c = np.r_[np.ones(n_c), np.zeros(n_w)]
|
|
293
|
+
conf = np.r_[np.full(n_c - 50, hi), np.full(50, lo), np.full(10, hi), np.full(n_w - 10, lo)] * scale
|
|
294
|
+
return c, conf
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def test_P1_unit_scale_mismatch_0_1_with_cut_90():
|
|
298
|
+
c, conf = _cont(scale=0.01)
|
|
299
|
+
with pytest.raises(ValueError, match="0-1 scale"):
|
|
300
|
+
screen(c, conf, cut=90)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def test_P2_unit_scale_mismatch_0_100_with_cut_09():
|
|
304
|
+
c, conf = _cont()
|
|
305
|
+
with pytest.raises(ValueError, match="0-100 scale"):
|
|
306
|
+
screen(c, conf, cut=0.9)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def test_P3_nan_cut():
|
|
310
|
+
c, conf = _cont()
|
|
311
|
+
with pytest.raises(ValueError):
|
|
312
|
+
screen(c, conf, cut=float("nan"))
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def test_P4_items_needed_integer_ceiling():
|
|
316
|
+
assert items_needed(0.9, 1.0, 0.9) == 350
|
|
317
|
+
assert items_needed(0.8, 1.0, 0.9) == 175
|
|
318
|
+
assert items_needed(0.9, 0.97, 0.9) == 1600
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def test_P5_flag_independent_of_alpha():
|
|
322
|
+
c, k = make(425, 30, 422, 27)
|
|
323
|
+
flags = [screen(c, k, alpha=a).flags for a in (0.05, 0.5)]
|
|
324
|
+
assert flags[0] == flags[1]
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
@pytest.mark.parametrize("a", [0, -1, 2, 1])
|
|
328
|
+
def test_P6_alpha_range(a):
|
|
329
|
+
c, k = make(450, 50, 400, 20)
|
|
330
|
+
with pytest.raises(ValueError):
|
|
331
|
+
screen(c, k, alpha=a)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def test_P7_argument_validation():
|
|
335
|
+
with pytest.raises(ValueError):
|
|
336
|
+
power_valid(452.5, .9, .95, .85)
|
|
337
|
+
with pytest.raises(ValueError):
|
|
338
|
+
power_valid(452, .9, 1.2, .5)
|
|
339
|
+
with pytest.raises(ValueError):
|
|
340
|
+
power_invalid(1.2, 50)
|
|
341
|
+
assert cli_main(["plan", "--accuracy", "0.9", "--true-L", "1.5"]) == 1
|
|
342
|
+
assert cli_main(["plan", "--accuracy", "0.9", "--power", "1.5"]) == 1
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def test_P8_unreadable_confidence_column(tmp_path, capsys):
|
|
346
|
+
p = tmp_path / "d.csv"
|
|
347
|
+
p.write_text("correct,confidence\n" + "\n".join("1,n/a" for _ in range(20)) + "\n")
|
|
348
|
+
assert cli_main(["run", "--data", str(p), "--cut", "90"]) == 1
|
|
349
|
+
assert "no confidence cell could be read" in capsys.readouterr().err
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def test_P8_percent_strings_are_read(tmp_path, capsys):
|
|
353
|
+
rows = ["1,95%"] * 400 + ["1,60%"] * 50 + ["0,95%"] * 10 + ["0,50%"] * 40
|
|
354
|
+
p = tmp_path / "d.csv"
|
|
355
|
+
p.write_text("correct,confidence\n" + "\n".join(rows) + "\n")
|
|
356
|
+
assert cli_main(["run", "--data", str(p), "--cut", "90", "--json"]) == 0
|
|
357
|
+
assert json.loads(capsys.readouterr().out)["n_scored"] == 500
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def test_P9_words_in_numeric_column_are_unreadable(tmp_path, capsys):
|
|
361
|
+
rows = ["1,95"] * 400 + ["1,60"] * 50 + ["0,95"] * 10 + ["0,50"] * 40
|
|
362
|
+
rows[0] = "1,high"
|
|
363
|
+
p = tmp_path / "d.csv"
|
|
364
|
+
p.write_text("correct,confidence\n" + "\n".join(rows) + "\n")
|
|
365
|
+
assert cli_main(["run", "--data", str(p), "--cut", "90", "--json"]) == 0
|
|
366
|
+
out = json.loads(capsys.readouterr().out)
|
|
367
|
+
assert out["n_missing"] == 1 and out["n_scored"] == 499
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def test_P10_no_scored_rows():
|
|
371
|
+
r = screen([1, 0, 1], [1, 1, 0], answered=[0, 0, 0])
|
|
372
|
+
assert r.tier == "Uncertifiable" and r.reason_code == "no_scored_rows"
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def test_P11_shapes_and_lengths():
|
|
376
|
+
c, k = make(450, 50, 400, 20)
|
|
377
|
+
with pytest.raises(ValueError):
|
|
378
|
+
screen(c, k, answered=np.ones(10))
|
|
379
|
+
with pytest.raises(ValueError):
|
|
380
|
+
screen(c.reshape(10, 50), k.reshape(10, 50))
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def test_P12_header_lines_skipped(tmp_path, capsys):
|
|
384
|
+
pc, pk = tmp_path / "c.txt", tmp_path / "k.txt"
|
|
385
|
+
pc.write_text("correct\n" + "\n".join(["1"] * 450 + ["0"] * 50) + "\n")
|
|
386
|
+
pk.write_text("keep\n" + "\n".join(["1"] * 400 + ["0"] * 50 + ["1"] * 10 + ["0"] * 40) + "\n")
|
|
387
|
+
assert cli_main(["run", "--correct", str(pc), "--confidence", str(pk), "--json"]) == 0
|
|
388
|
+
out = json.loads(capsys.readouterr().out)
|
|
389
|
+
assert out["n_items"] == 500 and out["n_missing"] == 0
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def test_P13_reason_code_vs_reason():
|
|
393
|
+
r = screen(*make(460, 40, 368, 16))
|
|
394
|
+
assert r.reason_code == "signal_detected" and r.reason.startswith("J > 0")
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def test_min_j_option():
|
|
398
|
+
c, k = make(4500, 500, 4300, 450) # J ~ .055, highly significant
|
|
399
|
+
assert screen(c, k).tier == "Valid"
|
|
400
|
+
r = screen(c, k, min_j=0.10)
|
|
401
|
+
assert r.tier == "Indeterminate" and r.reason_code == "sensitivity_below_minimum"
|
|
@@ -3,16 +3,17 @@ validity-screen
|
|
|
3
3
|
===============
|
|
4
4
|
|
|
5
5
|
Validity screen for LLM confidence signals: checks whether a model's confidence
|
|
6
|
-
measurably tracks correctness, and
|
|
7
|
-
blanket withdrawal
|
|
8
|
-
ECE, meta-d', selective prediction) are computed.
|
|
6
|
+
measurably tracks correctness, and whether a fixed response set (blanket
|
|
7
|
+
confidence, blanket withdrawal) or inversion makes it uninterpretable, before
|
|
8
|
+
downstream metrics (AUROC, ECE, meta-d', selective prediction) are computed.
|
|
9
9
|
|
|
10
10
|
Quick start::
|
|
11
11
|
|
|
12
12
|
from validity_screen import screen, binarise
|
|
13
13
|
|
|
14
|
-
result = screen(correct, keep, model_name="My Model")
|
|
15
|
-
|
|
14
|
+
result = screen(correct, keep, model_name="My Model") # binary confidence
|
|
15
|
+
result = screen(correct, conf_0_100, cut=90, model_name="My Model") # continuous
|
|
16
|
+
print(result.tier, result.reason, result.flags)
|
|
16
17
|
print(result.vrs_table())
|
|
17
18
|
|
|
18
19
|
Reference:
|
|
@@ -27,17 +28,21 @@ from validity_screen.core import (
|
|
|
27
28
|
binarise,
|
|
28
29
|
wilson_ci,
|
|
29
30
|
newcombe_diff_ci,
|
|
31
|
+
midp_signal,
|
|
30
32
|
errors_needed,
|
|
33
|
+
power_invalid,
|
|
34
|
+
power_valid,
|
|
31
35
|
items_needed,
|
|
32
36
|
ScreenResult,
|
|
33
37
|
IndexResult,
|
|
34
38
|
MIN_N,
|
|
35
39
|
)
|
|
36
40
|
|
|
37
|
-
__version__ = "0.
|
|
41
|
+
__version__ = "0.4.1"
|
|
38
42
|
__author__ = "Jon-Paul Cacioli"
|
|
39
43
|
|
|
40
44
|
__all__ = [
|
|
41
45
|
"screen", "screen_batch", "summary_table", "binarise", "wilson_ci", "newcombe_diff_ci",
|
|
42
|
-
"
|
|
46
|
+
"midp_signal", "errors_needed", "power_invalid", "power_valid", "items_needed",
|
|
47
|
+
"ScreenResult", "IndexResult", "MIN_N",
|
|
43
48
|
]
|