validity-screen 0.1.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- validity_screen-0.3.0/PKG-INFO +119 -0
- validity_screen-0.3.0/README.md +92 -0
- {validity_screen-0.1.0 → validity_screen-0.3.0}/pyproject.toml +4 -5
- validity_screen-0.3.0/tests/test_core.py +94 -0
- validity_screen-0.3.0/validity_screen/__init__.py +43 -0
- validity_screen-0.3.0/validity_screen/cli.py +122 -0
- validity_screen-0.3.0/validity_screen/core.py +373 -0
- validity_screen-0.3.0/validity_screen.egg-info/PKG-INFO +119 -0
- {validity_screen-0.1.0 → validity_screen-0.3.0}/validity_screen.egg-info/SOURCES.txt +1 -0
- validity_screen-0.1.0/PKG-INFO +0 -157
- validity_screen-0.1.0/README.md +0 -129
- validity_screen-0.1.0/validity_screen/__init__.py +0 -45
- validity_screen-0.1.0/validity_screen/cli.py +0 -162
- validity_screen-0.1.0/validity_screen/core.py +0 -465
- validity_screen-0.1.0/validity_screen.egg-info/PKG-INFO +0 -157
- {validity_screen-0.1.0 → validity_screen-0.3.0}/LICENSE +0 -0
- {validity_screen-0.1.0 → validity_screen-0.3.0}/setup.cfg +0 -0
- {validity_screen-0.1.0 → validity_screen-0.3.0}/validity_screen.egg-info/dependency_links.txt +0 -0
- {validity_screen-0.1.0 → validity_screen-0.3.0}/validity_screen.egg-info/entry_points.txt +0 -0
- {validity_screen-0.1.0 → validity_screen-0.3.0}/validity_screen.egg-info/requires.txt +0 -0
- {validity_screen-0.1.0 → validity_screen-0.3.0}/validity_screen.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: validity-screen
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Validity screen for LLM confidence signals (Cacioli, arXiv:2604.17714v2)
|
|
5
|
+
Author: Jon-Paul Cacioli
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/synthiumjp/validity-scaling-llm
|
|
8
|
+
Project-URL: Documentation, https://github.com/synthiumjp/validity-scaling-llm/tree/master/screen
|
|
9
|
+
Project-URL: Repository, https://github.com/synthiumjp/validity-scaling-llm
|
|
10
|
+
Project-URL: Issues, https://github.com/synthiumjp/validity-scaling-llm/issues
|
|
11
|
+
Keywords: llm,confidence,validity,metacognition,calibration,selective-prediction,screening,psychometrics,evaluation
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Requires-Python: >=3.9
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: numpy>=1.20
|
|
25
|
+
Requires-Dist: scipy>=1.7
|
|
26
|
+
Dynamic: license-file
|
|
27
|
+
|
|
28
|
+
# validity-screen
|
|
29
|
+
|
|
30
|
+
**Check whether an LLM's confidence signal tracks correctness, and which response set dominates it, before you build on it.**
|
|
31
|
+
|
|
32
|
+
[](https://pypi.org/project/validity-screen/)
|
|
33
|
+
[](https://opensource.org/licenses/MIT)
|
|
34
|
+
|
|
35
|
+
Implements the screen in:
|
|
36
|
+
|
|
37
|
+
> Cacioli, J. P. (2026). *Screen Before You Interpret: A Portable Validity Screen for Benchmark-Based LLM Confidence Signals.* arXiv:2604.17714v2.
|
|
38
|
+
|
|
39
|
+
> **Version 0.3.0 corrects version 0.1.** See [CHANGELOG.md](CHANGELOG.md). Results produced with 0.1 should be rerun.
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install validity-screen
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Quick start
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
import numpy as np
|
|
51
|
+
from validity_screen import screen, binarise
|
|
52
|
+
|
|
53
|
+
# item-level correctness and a binary confidence judgement (1 = KEEP / high)
|
|
54
|
+
result = screen(correct, keep, model_name="My Model")
|
|
55
|
+
print(result.tier) # Valid, Indeterminate, Invalid or Uncertifiable
|
|
56
|
+
print(result.vrs_table()) # full reporting table
|
|
57
|
+
|
|
58
|
+
# verbalized 0-100 confidence: choose the cut point in advance
|
|
59
|
+
result = screen(correct, binarise(confidence, 90), binarisation_threshold=">= 90")
|
|
60
|
+
|
|
61
|
+
# exclude declines, parse failures and pipeline defaults before scoring
|
|
62
|
+
result = screen(correct, keep, answered=answered)
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Command line:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
validity-screen run --data results.csv --correct-col is_correct --confidence-col keep
|
|
69
|
+
validity-screen run --data results.csv --confidence-col confidence --threshold 90
|
|
70
|
+
validity-screen run --data results.csv --answered-col parsed --json
|
|
71
|
+
validity-screen plan --accuracy 0.92 # items needed to certify blanket confidence
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## What it computes
|
|
75
|
+
|
|
76
|
+
From the correctness-by-confidence table (a = correct & high, b = incorrect & high, c = correct & low, d = incorrect & low):
|
|
77
|
+
|
|
78
|
+
| Index | Definition | Standard equivalent | Flag |
|
|
79
|
+
|---|---|---|---|
|
|
80
|
+
| **L** | b / (b + d): confidence kept on errors | false-positive rate of KEEP | >= .95 |
|
|
81
|
+
| **Fp** | c / (a + c): confidence withdrawn on correct answers | false-negative rate of KEEP | >= .50 |
|
|
82
|
+
| **RBS** | Fp - (1 - L) | minus Youden's J; binary AUROC = (1 - RBS)/2 | > 0 |
|
|
83
|
+
| **TRIN** | max(n_high, n_low) / N | modal-response share | >= .95, warning only |
|
|
84
|
+
| **r** | Pearson on 0/1 vectors | phi | reported |
|
|
85
|
+
|
|
86
|
+
Each index is computed only when its denominator reaches 10. A flagged index is **Invalid** if its interval clears the guard (Wilson lower bound > .40 for Fp, > .90 for L; Newcombe interval excluding 0 for RBS) and **Indeterminate** otherwise.
|
|
87
|
+
|
|
88
|
+
## Tiers
|
|
89
|
+
|
|
90
|
+
| Tier | Meaning |
|
|
91
|
+
|---|---|
|
|
92
|
+
| **Invalid** | A response set is established: blanket confidence, blanket withdrawal, or inversion. Do not interpret AUROC, ECE or selective prediction. |
|
|
93
|
+
| **Indeterminate** | A threshold is crossed but cannot be certified (often too few errors), or no response set is present but confidence does not measurably track correctness. |
|
|
94
|
+
| **Uncertifiable** | RBS cannot be estimated (fewer than 10 correct items or errors). |
|
|
95
|
+
| **Valid** | No response set, and the RBS interval lies below 0: confidence measurably tracks correctness. |
|
|
96
|
+
|
|
97
|
+
`require_signal=False` (CLI `--v02`) gives the 0.2 rule, under which Valid only meant that no flag fired. Every result also reports `tier_v02`.
|
|
98
|
+
|
|
99
|
+
## Things to know
|
|
100
|
+
|
|
101
|
+
- **Errors, not items, set the sample size.** Certifying blanket confidence needs 35 errors at L = 1.00, 71 at .97, 97 at .96 and 139 at .95. Use `errors_needed()` / `items_needed()`.
|
|
102
|
+
- **Continuous confidence needs a cut point chosen in advance.** Verdicts depend on it: verbalized confidence usually piles at 90-100, so at 50 most models look like blanket confidence. Screen at the threshold your deployment will use. Median splits are deprecated because their verdict depends on tie handling.
|
|
103
|
+
- **Valid is necessary, not sufficient.** It says the signal is non-degenerate and better than chance, not that it is good. Report r and AUROC alongside the tier.
|
|
104
|
+
- **Verdicts are specific to probe format, prompt and task.** Screen under the conditions the signal will be used in.
|
|
105
|
+
|
|
106
|
+
## Citation
|
|
107
|
+
|
|
108
|
+
```bibtex
|
|
109
|
+
@misc{cacioli2026screen,
|
|
110
|
+
author = {Cacioli, Jon-Paul},
|
|
111
|
+
title = {Screen Before You Interpret: A Portable Validity Screen for Benchmark-Based {LLM} Confidence Signals},
|
|
112
|
+
year = {2026},
|
|
113
|
+
howpublished = {arXiv:2604.17714v2}
|
|
114
|
+
}
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
## License
|
|
118
|
+
|
|
119
|
+
MIT.
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# validity-screen
|
|
2
|
+
|
|
3
|
+
**Check whether an LLM's confidence signal tracks correctness, and which response set dominates it, before you build on it.**
|
|
4
|
+
|
|
5
|
+
[](https://pypi.org/project/validity-screen/)
|
|
6
|
+
[](https://opensource.org/licenses/MIT)
|
|
7
|
+
|
|
8
|
+
Implements the screen in:
|
|
9
|
+
|
|
10
|
+
> Cacioli, J. P. (2026). *Screen Before You Interpret: A Portable Validity Screen for Benchmark-Based LLM Confidence Signals.* arXiv:2604.17714v2.
|
|
11
|
+
|
|
12
|
+
> **Version 0.3.0 corrects version 0.1.** See [CHANGELOG.md](CHANGELOG.md). Results produced with 0.1 should be rerun.
|
|
13
|
+
|
|
14
|
+
## Install
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install validity-screen
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Quick start
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
import numpy as np
|
|
24
|
+
from validity_screen import screen, binarise
|
|
25
|
+
|
|
26
|
+
# item-level correctness and a binary confidence judgement (1 = KEEP / high)
|
|
27
|
+
result = screen(correct, keep, model_name="My Model")
|
|
28
|
+
print(result.tier) # Valid, Indeterminate, Invalid or Uncertifiable
|
|
29
|
+
print(result.vrs_table()) # full reporting table
|
|
30
|
+
|
|
31
|
+
# verbalized 0-100 confidence: choose the cut point in advance
|
|
32
|
+
result = screen(correct, binarise(confidence, 90), binarisation_threshold=">= 90")
|
|
33
|
+
|
|
34
|
+
# exclude declines, parse failures and pipeline defaults before scoring
|
|
35
|
+
result = screen(correct, keep, answered=answered)
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Command line:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
validity-screen run --data results.csv --correct-col is_correct --confidence-col keep
|
|
42
|
+
validity-screen run --data results.csv --confidence-col confidence --threshold 90
|
|
43
|
+
validity-screen run --data results.csv --answered-col parsed --json
|
|
44
|
+
validity-screen plan --accuracy 0.92 # items needed to certify blanket confidence
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## What it computes
|
|
48
|
+
|
|
49
|
+
From the correctness-by-confidence table (a = correct & high, b = incorrect & high, c = correct & low, d = incorrect & low):
|
|
50
|
+
|
|
51
|
+
| Index | Definition | Standard equivalent | Flag |
|
|
52
|
+
|---|---|---|---|
|
|
53
|
+
| **L** | b / (b + d): confidence kept on errors | false-positive rate of KEEP | >= .95 |
|
|
54
|
+
| **Fp** | c / (a + c): confidence withdrawn on correct answers | false-negative rate of KEEP | >= .50 |
|
|
55
|
+
| **RBS** | Fp - (1 - L) | minus Youden's J; binary AUROC = (1 - RBS)/2 | > 0 |
|
|
56
|
+
| **TRIN** | max(n_high, n_low) / N | modal-response share | >= .95, warning only |
|
|
57
|
+
| **r** | Pearson on 0/1 vectors | phi | reported |
|
|
58
|
+
|
|
59
|
+
Each index is computed only when its denominator reaches 10. A flagged index is **Invalid** if its interval clears the guard (Wilson lower bound > .40 for Fp, > .90 for L; Newcombe interval excluding 0 for RBS) and **Indeterminate** otherwise.
|
|
60
|
+
|
|
61
|
+
## Tiers
|
|
62
|
+
|
|
63
|
+
| Tier | Meaning |
|
|
64
|
+
|---|---|
|
|
65
|
+
| **Invalid** | A response set is established: blanket confidence, blanket withdrawal, or inversion. Do not interpret AUROC, ECE or selective prediction. |
|
|
66
|
+
| **Indeterminate** | A threshold is crossed but cannot be certified (often too few errors), or no response set is present but confidence does not measurably track correctness. |
|
|
67
|
+
| **Uncertifiable** | RBS cannot be estimated (fewer than 10 correct items or errors). |
|
|
68
|
+
| **Valid** | No response set, and the RBS interval lies below 0: confidence measurably tracks correctness. |
|
|
69
|
+
|
|
70
|
+
`require_signal=False` (CLI `--v02`) gives the 0.2 rule, under which Valid only meant that no flag fired. Every result also reports `tier_v02`.
|
|
71
|
+
|
|
72
|
+
## Things to know
|
|
73
|
+
|
|
74
|
+
- **Errors, not items, set the sample size.** Certifying blanket confidence needs 35 errors at L = 1.00, 71 at .97, 97 at .96 and 139 at .95. Use `errors_needed()` / `items_needed()`.
|
|
75
|
+
- **Continuous confidence needs a cut point chosen in advance.** Verdicts depend on it: verbalized confidence usually piles at 90-100, so at 50 most models look like blanket confidence. Screen at the threshold your deployment will use. Median splits are deprecated because their verdict depends on tie handling.
|
|
76
|
+
- **Valid is necessary, not sufficient.** It says the signal is non-degenerate and better than chance, not that it is good. Report r and AUROC alongside the tier.
|
|
77
|
+
- **Verdicts are specific to probe format, prompt and task.** Screen under the conditions the signal will be used in.
|
|
78
|
+
|
|
79
|
+
## Citation
|
|
80
|
+
|
|
81
|
+
```bibtex
|
|
82
|
+
@misc{cacioli2026screen,
|
|
83
|
+
author = {Cacioli, Jon-Paul},
|
|
84
|
+
title = {Screen Before You Interpret: A Portable Validity Screen for Benchmark-Based {LLM} Confidence Signals},
|
|
85
|
+
year = {2026},
|
|
86
|
+
howpublished = {arXiv:2604.17714v2}
|
|
87
|
+
}
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
## License
|
|
91
|
+
|
|
92
|
+
MIT.
|
|
@@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "validity-screen"
|
|
7
|
-
version = "0.
|
|
8
|
-
description = "Validity
|
|
7
|
+
version = "0.3.0"
|
|
8
|
+
description = "Validity screen for LLM confidence signals (Cacioli, arXiv:2604.17714v2)"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {text = "MIT"}
|
|
11
|
-
requires-python = ">=3.
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
12
|
authors = [
|
|
13
13
|
{name = "Jon-Paul Cacioli"}
|
|
14
14
|
]
|
|
@@ -18,11 +18,10 @@ keywords = [
|
|
|
18
18
|
"psychometrics", "evaluation"
|
|
19
19
|
]
|
|
20
20
|
classifiers = [
|
|
21
|
-
"Development Status ::
|
|
21
|
+
"Development Status :: 4 - Beta",
|
|
22
22
|
"Intended Audience :: Science/Research",
|
|
23
23
|
"License :: OSI Approved :: MIT License",
|
|
24
24
|
"Programming Language :: Python :: 3",
|
|
25
|
-
"Programming Language :: Python :: 3.8",
|
|
26
25
|
"Programming Language :: Python :: 3.9",
|
|
27
26
|
"Programming Language :: Python :: 3.10",
|
|
28
27
|
"Programming Language :: Python :: 3.11",
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pytest
|
|
3
|
+
|
|
4
|
+
from validity_screen import screen, binarise, errors_needed, items_needed, screen_batch, summary_table
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def make(nc, ni, keep_c, keep_i):
|
|
8
|
+
correct = np.r_[np.ones(nc, int), np.zeros(ni, int)]
|
|
9
|
+
keep = np.r_[np.ones(keep_c), np.zeros(nc - keep_c), np.ones(keep_i), np.zeros(ni - keep_i)].astype(int)
|
|
10
|
+
return correct, keep
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def test_always_keep_invalid():
|
|
14
|
+
c, k = make(460, 40, 460, 40)
|
|
15
|
+
assert screen(c, k).tier == "Invalid"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_always_withdraw_invalid():
|
|
19
|
+
c, k = make(460, 40, 0, 0)
|
|
20
|
+
assert screen(c, k).tier == "Invalid"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_inverted_invalid():
|
|
24
|
+
c, k = make(460, 40, 92, 32) # keep 20% of correct, 80% of errors
|
|
25
|
+
assert screen(c, k).tier == "Invalid"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_good_monitor_valid():
|
|
29
|
+
c, k = make(460, 40, 368, 16) # keep 80% of correct, 40% of errors
|
|
30
|
+
r = screen(c, k)
|
|
31
|
+
assert r.tier == "Valid" and r.tier_v02 == "Valid"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_uninformative_not_valid_under_v03():
|
|
35
|
+
c, k = make(466, 58, 373, 46) # keep 80% of both
|
|
36
|
+
r = screen(c, k)
|
|
37
|
+
assert r.tier == "Indeterminate"
|
|
38
|
+
assert r.tier_v02 == "Valid"
|
|
39
|
+
assert screen(c, k, require_signal=False).tier == "Valid"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_high_L_few_errors_indeterminate():
|
|
43
|
+
c, k = make(476, 24, 457, 23) # L = .958 on 24 errors
|
|
44
|
+
r = screen(c, k)
|
|
45
|
+
assert r.tier == "Indeterminate" and r.L.flag == "indeterminate"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_not_estimable_with_few_errors():
|
|
49
|
+
c, k = make(446, 6, 24, 0) # R1-like: 6 errors, Fp = .946
|
|
50
|
+
r = screen(c, k)
|
|
51
|
+
assert r.L.flag == "not_estimable" and r.rbs.flag == "not_estimable"
|
|
52
|
+
assert r.tier == "Invalid" # decided by Fp
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_exclusions():
|
|
56
|
+
c, k = make(460, 40, 368, 16)
|
|
57
|
+
ans = np.ones(500, int); ans[:10] = 0
|
|
58
|
+
r = screen(c, k, answered=ans)
|
|
59
|
+
assert r.n_excluded == 10 and r.n_scored == 490
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def test_identities():
|
|
63
|
+
rng = np.random.default_rng(0)
|
|
64
|
+
from sklearn.metrics import roc_auc_score
|
|
65
|
+
for _ in range(100):
|
|
66
|
+
cor = (rng.random(400) < rng.uniform(.6, .95)).astype(int)
|
|
67
|
+
conf = np.where(rng.random(400) < .7, cor, 1 - cor)
|
|
68
|
+
conf = np.where(rng.random(400) < .5, 1, conf)
|
|
69
|
+
r = screen(cor, conf)
|
|
70
|
+
if np.isnan(r.rbs.value):
|
|
71
|
+
continue
|
|
72
|
+
assert abs(r.auroc_binary - roc_auc_score(cor, conf)) < 1e-12
|
|
73
|
+
assert abs(r.youden_j + r.rbs.value) < 1e-12
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_errors_needed():
|
|
77
|
+
assert [errors_needed(x) for x in (1.0, .97, .96, .95)] == [35, 71, 97, 139]
|
|
78
|
+
assert errors_needed(.90) is None
|
|
79
|
+
assert items_needed(0.92, 1.0) == 438
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_binarise_and_median_deprecated():
|
|
83
|
+
x = np.array([10, 50, 90, 100])
|
|
84
|
+
assert binarise(x, 90).tolist() == [False, False, True, True]
|
|
85
|
+
with pytest.warns(DeprecationWarning):
|
|
86
|
+
binarise(x, method="median")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_batch_and_summary():
|
|
90
|
+
c, k = make(460, 40, 368, 16)
|
|
91
|
+
res = screen_batch({"a": {"correct": c, "confidence": k}, "b": {"correct": c, "confidence": np.ones(500)}})
|
|
92
|
+
s = summary_table(res)
|
|
93
|
+
assert "Invalid" in s and "Valid" in s
|
|
94
|
+
assert "VRS TABLE" in res[0].vrs_table()
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""
|
|
2
|
+
validity-screen
|
|
3
|
+
===============
|
|
4
|
+
|
|
5
|
+
Validity screen for LLM confidence signals: checks whether a model's confidence
|
|
6
|
+
measurably tracks correctness, and which response set (blanket confidence,
|
|
7
|
+
blanket withdrawal, inversion) dominates it, before downstream metrics (AUROC,
|
|
8
|
+
ECE, meta-d', selective prediction) are computed.
|
|
9
|
+
|
|
10
|
+
Quick start::
|
|
11
|
+
|
|
12
|
+
from validity_screen import screen, binarise
|
|
13
|
+
|
|
14
|
+
result = screen(correct, keep, model_name="My Model")
|
|
15
|
+
print(result.tier) # 'Valid', 'Indeterminate', 'Invalid' or 'Uncertifiable'
|
|
16
|
+
print(result.vrs_table())
|
|
17
|
+
|
|
18
|
+
Reference:
|
|
19
|
+
Cacioli, J. P. (2026). Screen Before You Interpret: A Portable Validity Screen
|
|
20
|
+
for Benchmark-Based LLM Confidence Signals. arXiv:2604.17714v2.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from validity_screen.core import (
|
|
24
|
+
screen,
|
|
25
|
+
screen_batch,
|
|
26
|
+
summary_table,
|
|
27
|
+
binarise,
|
|
28
|
+
wilson_ci,
|
|
29
|
+
newcombe_diff_ci,
|
|
30
|
+
errors_needed,
|
|
31
|
+
items_needed,
|
|
32
|
+
ScreenResult,
|
|
33
|
+
IndexResult,
|
|
34
|
+
MIN_N,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
__version__ = "0.3.0"
|
|
38
|
+
__author__ = "Jon-Paul Cacioli"
|
|
39
|
+
|
|
40
|
+
__all__ = [
|
|
41
|
+
"screen", "screen_batch", "summary_table", "binarise", "wilson_ci", "newcombe_diff_ci",
|
|
42
|
+
"errors_needed", "items_needed", "ScreenResult", "IndexResult", "MIN_N",
|
|
43
|
+
]
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command-line interface for validity-screen.
|
|
3
|
+
|
|
4
|
+
validity-screen run --data results.csv --correct-col is_correct --confidence-col keep
|
|
5
|
+
validity-screen run --data results.csv --confidence-col confidence --threshold 90
|
|
6
|
+
validity-screen plan --accuracy 0.92 --L 1.0
|
|
7
|
+
validity-screen version
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import csv
|
|
12
|
+
import json
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
|
|
17
|
+
TRUE = ("true", "1", "1.0", "yes", "keep", "bet", "high")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _parse_conf(vals):
|
|
21
|
+
"""Return (array, is_binary). Numeric strings become floats; words become 0/1."""
|
|
22
|
+
out, numeric = [], True
|
|
23
|
+
for v in vals:
|
|
24
|
+
s = str(v).strip().lower()
|
|
25
|
+
try:
|
|
26
|
+
out.append(float(s))
|
|
27
|
+
except ValueError:
|
|
28
|
+
numeric = False
|
|
29
|
+
out.append(1.0 if s in TRUE else 0.0)
|
|
30
|
+
arr = np.array(out, dtype=float)
|
|
31
|
+
return arr, (not numeric) or bool(np.all(np.isin(arr, [0.0, 1.0])))
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _load(args):
|
|
35
|
+
if args.data:
|
|
36
|
+
with open(args.data, newline="") as f:
|
|
37
|
+
rows = list(csv.DictReader(f))
|
|
38
|
+
correct = np.array([str(r[args.correct_col]).strip().lower() in TRUE for r in rows])
|
|
39
|
+
conf_raw = [r[args.confidence_col] for r in rows]
|
|
40
|
+
answered = None
|
|
41
|
+
if args.answered_col:
|
|
42
|
+
answered = np.array([str(r[args.answered_col]).strip().lower() in TRUE for r in rows])
|
|
43
|
+
return correct, conf_raw, answered
|
|
44
|
+
if args.correct and args.confidence:
|
|
45
|
+
def col(p):
|
|
46
|
+
with open(p) as f:
|
|
47
|
+
return [ln.strip() for ln in f if ln.strip()]
|
|
48
|
+
return np.array([v.lower() in TRUE for v in col(args.correct)]), col(args.confidence), None
|
|
49
|
+
raise SystemExit("Error: provide --data, or both --correct and --confidence.")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def main(argv=None):
|
|
53
|
+
p = argparse.ArgumentParser(prog="validity-screen",
|
|
54
|
+
description="Validity screen for LLM confidence signals.")
|
|
55
|
+
sub = p.add_subparsers(dest="command")
|
|
56
|
+
|
|
57
|
+
r = sub.add_parser("run", help="Screen one model's data.")
|
|
58
|
+
r.add_argument("--data", help="CSV with correctness and confidence columns.")
|
|
59
|
+
r.add_argument("--correct", help="File of correctness labels, one per line.")
|
|
60
|
+
r.add_argument("--confidence", help="File of confidence values, one per line.")
|
|
61
|
+
r.add_argument("--correct-col", default="correct")
|
|
62
|
+
r.add_argument("--confidence-col", default="confidence")
|
|
63
|
+
r.add_argument("--answered-col", default=None,
|
|
64
|
+
help="Optional column; rows that are not true (declines, parse failures, "
|
|
65
|
+
"pipeline defaults) are excluded before scoring.")
|
|
66
|
+
r.add_argument("--threshold", type=float, default=None,
|
|
67
|
+
help="Cut point for continuous confidence (confidence >= threshold is high). "
|
|
68
|
+
"Required for continuous confidence; choose it in advance.")
|
|
69
|
+
r.add_argument("--model-name", default="")
|
|
70
|
+
r.add_argument("--benchmark", default="")
|
|
71
|
+
r.add_argument("--v02", action="store_true",
|
|
72
|
+
help="Use the version 0.2 rule (Valid = no flag fired).")
|
|
73
|
+
r.add_argument("--json", action="store_true", help="JSON output instead of the VRS Table.")
|
|
74
|
+
|
|
75
|
+
pl = sub.add_parser("plan", help="Items/errors needed to certify blanket confidence.")
|
|
76
|
+
pl.add_argument("--accuracy", type=float, required=True)
|
|
77
|
+
pl.add_argument("--L", type=float, default=1.0, help="Observed L to certify (default 1.0).")
|
|
78
|
+
|
|
79
|
+
sub.add_parser("version", help="Print version.")
|
|
80
|
+
args = p.parse_args(argv)
|
|
81
|
+
|
|
82
|
+
if args.command == "version":
|
|
83
|
+
from validity_screen import __version__
|
|
84
|
+
print(f"validity-screen {__version__}")
|
|
85
|
+
return 0
|
|
86
|
+
if args.command == "plan":
|
|
87
|
+
from validity_screen import errors_needed, items_needed
|
|
88
|
+
e, n = errors_needed(args.L), items_needed(args.accuracy, args.L)
|
|
89
|
+
if e is None:
|
|
90
|
+
print(f"L = {args.L} cannot be certified above the .90 guard.")
|
|
91
|
+
else:
|
|
92
|
+
print(f"To certify L = {args.L} (Wilson lower bound > .90): {e} errors, "
|
|
93
|
+
f"about {n} items at accuracy {args.accuracy}.")
|
|
94
|
+
return 0
|
|
95
|
+
if args.command != "run":
|
|
96
|
+
p.print_help()
|
|
97
|
+
return 0
|
|
98
|
+
|
|
99
|
+
from validity_screen import screen, binarise
|
|
100
|
+
try:
|
|
101
|
+
correct, conf_raw, answered = _load(args)
|
|
102
|
+
conf, is_binary = _parse_conf(conf_raw)
|
|
103
|
+
if is_binary:
|
|
104
|
+
confidence, cut = conf.astype(bool), "N/A (binary)"
|
|
105
|
+
else:
|
|
106
|
+
if args.threshold is None:
|
|
107
|
+
raise SystemExit("Error: confidence is continuous; pass --threshold (a cut point chosen "
|
|
108
|
+
"in advance, ideally the deployment threshold). Median splits are not "
|
|
109
|
+
"supported because their verdict depends on tie handling.")
|
|
110
|
+
confidence, cut = binarise(conf, args.threshold), f">= {args.threshold:g}"
|
|
111
|
+
res = screen(correct, confidence, answered=answered, model_name=args.model_name,
|
|
112
|
+
benchmark_name=args.benchmark, binarisation_threshold=cut,
|
|
113
|
+
require_signal=not args.v02)
|
|
114
|
+
except (FileNotFoundError, KeyError) as e:
|
|
115
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
116
|
+
return 1
|
|
117
|
+
print(json.dumps(res.to_dict(), indent=2) if args.json else res.vrs_table())
|
|
118
|
+
return 0
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
if __name__ == "__main__":
|
|
122
|
+
sys.exit(main())
|