caat-annotation 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- caat_annotation-0.2.0/LICENSE +21 -0
- caat_annotation-0.2.0/PKG-INFO +221 -0
- caat_annotation-0.2.0/README.md +190 -0
- caat_annotation-0.2.0/pyproject.toml +44 -0
- caat_annotation-0.2.0/setup.cfg +4 -0
- caat_annotation-0.2.0/src/caat/__init__.py +9 -0
- caat_annotation-0.2.0/src/caat/adapters.py +23 -0
- caat_annotation-0.2.0/src/caat/audit.py +131 -0
- caat_annotation-0.2.0/src/caat/calibrate.py +328 -0
- caat_annotation-0.2.0/src/caat/cli.py +761 -0
- caat_annotation-0.2.0/src/caat/collect.py +205 -0
- caat_annotation-0.2.0/src/caat/evaluate.py +66 -0
- caat_annotation-0.2.0/src/caat/features.py +304 -0
- caat_annotation-0.2.0/src/caat/html_report.py +130 -0
- caat_annotation-0.2.0/src/caat/metrics.py +196 -0
- caat_annotation-0.2.0/src/caat/plots.py +223 -0
- caat_annotation-0.2.0/src/caat/plugins.py +32 -0
- caat_annotation-0.2.0/src/caat/policy.py +127 -0
- caat_annotation-0.2.0/src/caat/queue.py +75 -0
- caat_annotation-0.2.0/src/caat/registry.py +71 -0
- caat_annotation-0.2.0/src/caat/renderers.py +64 -0
- caat_annotation-0.2.0/src/caat/report.py +209 -0
- caat_annotation-0.2.0/src/caat/rules.py +127 -0
- caat_annotation-0.2.0/src/caat/scores.py +231 -0
- caat_annotation-0.2.0/src/caat/select.py +88 -0
- caat_annotation-0.2.0/src/caat/table.py +236 -0
- caat_annotation-0.2.0/src/caat_annotation.egg-info/PKG-INFO +221 -0
- caat_annotation-0.2.0/src/caat_annotation.egg-info/SOURCES.txt +40 -0
- caat_annotation-0.2.0/src/caat_annotation.egg-info/dependency_links.txt +1 -0
- caat_annotation-0.2.0/src/caat_annotation.egg-info/entry_points.txt +2 -0
- caat_annotation-0.2.0/src/caat_annotation.egg-info/requires.txt +8 -0
- caat_annotation-0.2.0/src/caat_annotation.egg-info/top_level.txt +1 -0
- caat_annotation-0.2.0/tests/test_caat_audit.py +215 -0
- caat_annotation-0.2.0/tests/test_caat_calibration.py +143 -0
- caat_annotation-0.2.0/tests/test_caat_evaluate.py +113 -0
- caat_annotation-0.2.0/tests/test_caat_features.py +92 -0
- caat_annotation-0.2.0/tests/test_caat_policy.py +131 -0
- caat_annotation-0.2.0/tests/test_caat_scores.py +75 -0
- caat_annotation-0.2.0/tests/test_caat_selection.py +132 -0
- caat_annotation-0.2.0/tests/test_caat_use.py +133 -0
- caat_annotation-0.2.0/tests/test_readme_reports.py +223 -0
- caat_annotation-0.2.0/tests/test_replay_benchmarks.py +57 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025-2026 The CAAT contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: caat-annotation
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Score, calibrate, evaluate, and triage saved model predictions
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/Daveyuwang/caat
|
|
7
|
+
Project-URL: Documentation, https://github.com/Daveyuwang/caat/blob/main/README.md
|
|
8
|
+
Project-URL: Issues, https://github.com/Daveyuwang/caat/issues
|
|
9
|
+
Keywords: selective prediction,risk control,LLM evaluation,human review,calibration
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: ruff==0.14.14; extra == "dev"
|
|
26
|
+
Provides-Extra: models
|
|
27
|
+
Requires-Dist: torch>=2.6; extra == "models"
|
|
28
|
+
Requires-Dist: transformers<6,>=4.57; extra == "models"
|
|
29
|
+
Requires-Dist: accelerate>=1.1; extra == "models"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# Confidence-Aware Annotation Toolkit
|
|
33
|
+
|
|
34
|
+
[](https://github.com/Daveyuwang/caat/actions/workflows/ci.yml)
|
|
35
|
+
|
|
36
|
+
CAAT scores, calibrates, evaluates, and triages saved model predictions.
|
|
37
|
+
Compare confidence signals, measure correctness probabilities, and choose
|
|
38
|
+
which outputs to accept automatically or send for review. The analysis
|
|
39
|
+
commands run on a CPU with Python 3.10 or later.
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
git clone https://github.com/Daveyuwang/caat.git
|
|
43
|
+
cd caat
|
|
44
|
+
python -m pip install .
|
|
45
|
+
caat evaluate examples/ticket_triage.csv --id-column ticket_id --prediction-column model_route --reference-column expected_route --score-column confidence --group-column inbox --threshold 0.75 --probability --output-dir replay/tickets
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The example accepts 3 of 5 tickets and makes 1 error. CAAT writes `metrics.json`
|
|
49
|
+
with overall and per-group counts, AUROC, area under the risk–coverage curve
|
|
50
|
+
(AURC), and normalized AURC gain. It writes `risk_coverage.csv` with the
|
|
51
|
+
results at each distinct score threshold. A normalized gain of 0 matches
|
|
52
|
+
random ordering and 1 matches ordering all correct predictions first.
|
|
53
|
+
|
|
54
|
+
## CSV input
|
|
55
|
+
|
|
56
|
+
Each row needs a unique `id` and a finite `score`. Larger scores indicate
|
|
57
|
+
greater confidence. Supply `prediction` and `reference` for exact label
|
|
58
|
+
matching, or use `--correct-column NAME` for a grader's 0/1 outcome. Map
|
|
59
|
+
different headers with the flags above. An optional `group` column gives
|
|
60
|
+
per-group metrics.
|
|
61
|
+
|
|
62
|
+
CAAT accepts every row with a score at or above `--threshold`. Use
|
|
63
|
+
`--probability` to report Brier score, ECE, ACE, and reliability bins for
|
|
64
|
+
probabilities of correctness.
|
|
65
|
+
The review commands print a short summary. Add `--json` for machine-readable
|
|
66
|
+
stdout or `--output FILE` to save the JSON report. The CLI and Python API
|
|
67
|
+
use the standard library. [CSV input](https://github.com/Daveyuwang/caat/blob/main/docs/guide/csv-format.md) · [Report fields](https://github.com/Daveyuwang/caat/blob/main/docs/guide/reports.md)
|
|
68
|
+
|
|
69
|
+
Use `caat calibrate fit` to save a temperature, Platt, isotonic, or histogram
|
|
70
|
+
map from labeled calibration rows. Use `caat calibrate apply` to add its
|
|
71
|
+
probabilities to a held-out CSV. Add `--html report.html` to `evaluate`,
|
|
72
|
+
`audit`, or `policy` for a self-contained report.
|
|
73
|
+
[Calibration examples](https://github.com/Daveyuwang/caat/blob/main/docs/guide/calibration.md)
|
|
74
|
+
|
|
75
|
+

|
|
76
|
+
|
|
77
|
+
## Choose and check a cutoff in one command
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
caat policy examples/ifeval_graders.csv --score-column mean_token_logprob --correct-column strict_pass --check-column helm_strict_pass --risk-target 0.4 --plot replay/policy.svg
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
`caat policy` splits one labeled CSV into design, calibration, and held-out
|
|
84
|
+
rows. It selects a cutoff on the calibration rows and applies it unchanged
|
|
85
|
+
to the held-out rows. The example holds 541 saved IFEval responses graded by
|
|
86
|
+
Google's strict grader and by HELM. The command selects
|
|
87
|
+
`-0.43964836263206303` under the Google grader. On the 217 held-out rows the
|
|
88
|
+
policy accepts 144 responses, with 33 errors (22.9%) under Google and 45
|
|
89
|
+
(31.2%) under HELM. Both meet the 40% target.
|
|
90
|
+
|
|
91
|
+

|
|
92
|
+
|
|
93
|
+
A `--check-column` grader is reported on held-out rows and takes no part in
|
|
94
|
+
selection. Pass it as a second `--correct-column` to require the target
|
|
95
|
+
under both graders. The cutoff then rises to `-0.3327795407672605` and the
|
|
96
|
+
policy accepts 92 held-out responses, with 19 and 23 errors.
|
|
97
|
+
|
|
98
|
+
The summary reports AUROC for each grader on the calibration rows and says
|
|
99
|
+
when the score ranks no better than random. The split sorts IDs, shuffles
|
|
100
|
+
with `--seed` (default 42), and assigns about 20%, 40%, and 40% of the rows.
|
|
101
|
+
`--rule` and `--failure-rate` work as in `caat select`. Add `--json` for the
|
|
102
|
+
full report. The command returns 0 if the held-out rows meet the target
|
|
103
|
+
under every listed grader, 1 if no cutoff qualifies or a grader exceeds the
|
|
104
|
+
target, and 2 for invalid input.
|
|
105
|
+
|
|
106
|
+
## Compare graders on the same outputs
|
|
107
|
+
|
|
108
|
+
Run a frozen threshold against separate grader exports. CAAT matches rows
|
|
109
|
+
by ID and checks that each grader scored the same saved output.
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
caat audit examples/audit/predictions.csv --output-key-column output_key --grader-file reference=examples/audit/reference.csv --grader-file rubric=examples/audit/rubric.csv --threshold 0.75 --risk-target 0.4
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
The sample accepts three tickets. The reference grader counts one error,
|
|
116
|
+
and the rubric grader counts two. Use `--json` for the changed
|
|
117
|
+
decision, discordant IDs, and per-group counts. For each alternate grader,
|
|
118
|
+
`overall_risk_change` measures the shift on all outputs and
|
|
119
|
+
`excess_risk_change` subtracts that shift from the accepted-set shift.
|
|
120
|
+
Add `--bootstrap 2000 --seed 42` for paired 95% intervals at the frozen
|
|
121
|
+
cutoff. The sample exits 1 because the rubric grader exceeds the 40%
|
|
122
|
+
risk target. A policy with zero accepted rows also exits 1 if you supply
|
|
123
|
+
a target. Invalid input returns 2.
|
|
124
|
+
|
|
125
|
+
For separate grader files, include `id`, `output_sha256`, and a 0/1
|
|
126
|
+
`correct` column in each export. The prediction file supplies the same
|
|
127
|
+
IDs and output hashes plus the score. Use `--output-key-column` if your
|
|
128
|
+
pipeline already records another output identity. If the results are
|
|
129
|
+
already aligned in one CSV, repeat `--correct-column NAME` for each grader.
|
|
130
|
+
The first grader is the reference for directional differences.
|
|
131
|
+
|
|
132
|
+
If a prediction CSV contains multiple runs for the same IDs, select one
|
|
133
|
+
with `--prediction-filter COLUMN=VALUE`.
|
|
134
|
+
[Real routing example and CI setup](https://github.com/Daveyuwang/caat/blob/main/docs/guide/ci-audit.md)
|
|
135
|
+
|
|
136
|
+
## Select a threshold on calibration data
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
caat select examples/ifeval_strict.csv --score-column mean_token_logprob --correct-column strict_pass --thresholds -0.20 -0.35 -0.60 --risk-target 0.4 --rule empirical
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
The example selects `-0.35`, accepting 12 of 24 responses with four
|
|
143
|
+
errors on the calibration sample. Supply a fixed list with `--thresholds`
|
|
144
|
+
or use `--design DESIGN.csv` to derive candidates from score deciles on a
|
|
145
|
+
separate file. Repeat `--correct-column` to require the target under every
|
|
146
|
+
grader. The JSON includes per-candidate counts and `selected_threshold`.
|
|
147
|
+
The command returns 0 if it selects a cutoff, 1 if none qualifies, and 2
|
|
148
|
+
for invalid input.
|
|
149
|
+
|
|
150
|
+
For statistical risk control, use `--rule ltt-bonferroni` (default),
|
|
151
|
+
`--rule ltt-holm`, or `--rule ltt-fixed-sequence`. These rules assume
|
|
152
|
+
independent calibration rows from the target population and a candidate
|
|
153
|
+
grid fixed before calibration. Holm's rule tests candidates in order of
|
|
154
|
+
increasing p-value. [Holm's step-down procedure](https://www.jstor.org/stable/4615733)
|
|
155
|
+
`--failure-rate` defaults to 0.05. Fixed-sequence testing proceeds from
|
|
156
|
+
the strictest threshold and stops at the first failure. The empirical rule
|
|
157
|
+
has no statistical guarantee. Apply the selected cutoff unchanged to
|
|
158
|
+
held-out data with `caat evaluate` or `caat audit`.
|
|
159
|
+
|
|
160
|
+
## Score saved responses
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
caat features examples/token_scores.jsonl --output replay/token_scores.csv
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
The input supplies response text and token log probabilities. The CSV
|
|
167
|
+
contains one row per response, with scores you can use as `--score-column`
|
|
168
|
+
values. [JSONL format and available scores](https://github.com/Daveyuwang/caat/blob/main/docs/guide/features.md)
|
|
169
|
+
|
|
170
|
+
Use `caat scores` to compare candidate columns at matched coverage budgets
|
|
171
|
+
and pick a score on the design split before calibrating a threshold.
|
|
172
|
+
[Score comparison guide](https://github.com/Daveyuwang/caat/blob/main/docs/guide/scores.md)
|
|
173
|
+
|
|
174
|
+
Use `caat queue` to order saved predictions for review under a row budget.
|
|
175
|
+
It can cap the number of rows per group and compare error yield with a
|
|
176
|
+
random queue. [Queue guide](https://github.com/Daveyuwang/caat/blob/main/docs/guide/queue.md) ·
|
|
177
|
+
[Course replay](https://github.com/Daveyuwang/caat/blob/main/docs/research/benchmarks.md#review-queues)
|
|
178
|
+
|
|
179
|
+
## CPU benchmarks
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
python scripts/replay_benchmarks.py
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
The command replays the bundled 24-response samples for IFEval, MMLU-Pro, and
|
|
186
|
+
TriviaQA and writes `replay/benchmarks/summary.csv`. The larger HELM comparison
|
|
187
|
+
evaluates sequence probability and perplexity rankings and tests empirical
|
|
188
|
+
threshold selection against Learn then Test.
|
|
189
|
+
[Sources, results, and reproduction](https://github.com/Daveyuwang/caat/blob/main/docs/research/benchmarks.md) ·
|
|
190
|
+
[WildBench and Omni-MATH grader replay](https://github.com/Daveyuwang/caat/blob/main/docs/research/judge-study.md)
|
|
191
|
+
|
|
192
|
+
The local Qwen2.5-7B study compares token scores, self-reports, and sample
|
|
193
|
+
agreement on TriviaQA and MMLU-Pro. It includes a CPU replay and measured
|
|
194
|
+
collection costs from an RTX 5080.
|
|
195
|
+
[Qwen confidence benchmark](https://github.com/Daveyuwang/caat/blob/main/docs/research/black-box-benchmark.md)
|
|
196
|
+
|
|
197
|
+
## Development
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
python -m pip install -e .
|
|
201
|
+
python -m unittest discover -s tests
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
The tests include the bundled benchmark replay. CI runs them on Python 3.10
|
|
205
|
+
through 3.14 on Linux and on 3.10 and 3.14 on Windows, runs the README
|
|
206
|
+
examples, and builds and installs the wheel.
|
|
207
|
+
|
|
208
|
+
[Evaluation method and related work](https://github.com/Daveyuwang/caat/blob/main/docs/research/method.md)
|
|
209
|
+
|
|
210
|
+
`caat plugins` lists built-in and installed components. The optional
|
|
211
|
+
Transformers backend collects model outputs for later scoring and replay.
|
|
212
|
+
[Plugin guide](https://github.com/Daveyuwang/caat/blob/main/docs/guide/plugins.md) ·
|
|
213
|
+
[Local model collection](https://github.com/Daveyuwang/caat/blob/main/docs/guide/collect.md)
|
|
214
|
+
|
|
215
|
+
[Contributing](https://github.com/Daveyuwang/caat/blob/main/CONTRIBUTING.md) ·
|
|
216
|
+
[Roadmap](https://github.com/Daveyuwang/caat/blob/main/docs/ROADMAP.md) ·
|
|
217
|
+
[Changelog](https://github.com/Daveyuwang/caat/blob/main/CHANGELOG.md)
|
|
218
|
+
|
|
219
|
+
## License
|
|
220
|
+
|
|
221
|
+
MIT. See [LICENSE](https://github.com/Daveyuwang/caat/blob/main/LICENSE).
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
# Confidence-Aware Annotation Toolkit
|
|
2
|
+
|
|
3
|
+
[](https://github.com/Daveyuwang/caat/actions/workflows/ci.yml)
|
|
4
|
+
|
|
5
|
+
CAAT scores, calibrates, evaluates, and triages saved model predictions.
|
|
6
|
+
Compare confidence signals, measure correctness probabilities, and choose
|
|
7
|
+
which outputs to accept automatically or send for review. The analysis
|
|
8
|
+
commands run on a CPU with Python 3.10 or later.
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
git clone https://github.com/Daveyuwang/caat.git
|
|
12
|
+
cd caat
|
|
13
|
+
python -m pip install .
|
|
14
|
+
caat evaluate examples/ticket_triage.csv --id-column ticket_id --prediction-column model_route --reference-column expected_route --score-column confidence --group-column inbox --threshold 0.75 --probability --output-dir replay/tickets
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
The example accepts 3 of 5 tickets and makes 1 error. CAAT writes `metrics.json`
|
|
18
|
+
with overall and per-group counts, AUROC, area under the risk–coverage curve
|
|
19
|
+
(AURC), and normalized AURC gain. It writes `risk_coverage.csv` with the
|
|
20
|
+
results at each distinct score threshold. A normalized gain of 0 matches
|
|
21
|
+
random ordering and 1 matches ordering all correct predictions first.
|
|
22
|
+
|
|
23
|
+
## CSV input
|
|
24
|
+
|
|
25
|
+
Each row needs a unique `id` and a finite `score`. Larger scores indicate
|
|
26
|
+
greater confidence. Supply `prediction` and `reference` for exact label
|
|
27
|
+
matching, or use `--correct-column NAME` for a grader's 0/1 outcome. Map
|
|
28
|
+
different headers with the flags above. An optional `group` column gives
|
|
29
|
+
per-group metrics.
|
|
30
|
+
|
|
31
|
+
CAAT accepts every row with a score at or above `--threshold`. Use
|
|
32
|
+
`--probability` to report Brier score, ECE, ACE, and reliability bins for
|
|
33
|
+
probabilities of correctness.
|
|
34
|
+
The review commands print a short summary. Add `--json` for machine-readable
|
|
35
|
+
stdout or `--output FILE` to save the JSON report. The CLI and Python API
|
|
36
|
+
use the standard library. [CSV input](https://github.com/Daveyuwang/caat/blob/main/docs/guide/csv-format.md) · [Report fields](https://github.com/Daveyuwang/caat/blob/main/docs/guide/reports.md)
|
|
37
|
+
|
|
38
|
+
Use `caat calibrate fit` to save a temperature, Platt, isotonic, or histogram
|
|
39
|
+
map from labeled calibration rows. Use `caat calibrate apply` to add its
|
|
40
|
+
probabilities to a held-out CSV. Add `--html report.html` to `evaluate`,
|
|
41
|
+
`audit`, or `policy` for a self-contained report.
|
|
42
|
+
[Calibration examples](https://github.com/Daveyuwang/caat/blob/main/docs/guide/calibration.md)
|
|
43
|
+
|
|
44
|
+

|
|
45
|
+
|
|
46
|
+
## Choose and check a cutoff in one command
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
caat policy examples/ifeval_graders.csv --score-column mean_token_logprob --correct-column strict_pass --check-column helm_strict_pass --risk-target 0.4 --plot replay/policy.svg
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
`caat policy` splits one labeled CSV into design, calibration, and held-out
|
|
53
|
+
rows. It selects a cutoff on the calibration rows and applies it unchanged
|
|
54
|
+
to the held-out rows. The example holds 541 saved IFEval responses graded by
|
|
55
|
+
Google's strict grader and by HELM. The command selects
|
|
56
|
+
`-0.43964836263206303` under the Google grader. On the 217 held-out rows the
|
|
57
|
+
policy accepts 144 responses, with 33 errors (22.9%) under Google and 45
|
|
58
|
+
(31.2%) under HELM. Both meet the 40% target.
|
|
59
|
+
|
|
60
|
+

|
|
61
|
+
|
|
62
|
+
A `--check-column` grader is reported on held-out rows and takes no part in
|
|
63
|
+
selection. Pass it as a second `--correct-column` to require the target
|
|
64
|
+
under both graders. The cutoff then rises to `-0.3327795407672605` and the
|
|
65
|
+
policy accepts 92 held-out responses, with 19 and 23 errors.
|
|
66
|
+
|
|
67
|
+
The summary reports AUROC for each grader on the calibration rows and says
|
|
68
|
+
when the score ranks no better than random. The split sorts IDs, shuffles
|
|
69
|
+
with `--seed` (default 42), and assigns about 20%, 40%, and 40% of the rows.
|
|
70
|
+
`--rule` and `--failure-rate` work as in `caat select`. Add `--json` for the
|
|
71
|
+
full report. The command returns 0 if the held-out rows meet the target
|
|
72
|
+
under every listed grader, 1 if no cutoff qualifies or a grader exceeds the
|
|
73
|
+
target, and 2 for invalid input.
|
|
74
|
+
|
|
75
|
+
## Compare graders on the same outputs
|
|
76
|
+
|
|
77
|
+
Run a frozen threshold against separate grader exports. CAAT matches rows
|
|
78
|
+
by ID and checks that each grader scored the same saved output.
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
caat audit examples/audit/predictions.csv --output-key-column output_key --grader-file reference=examples/audit/reference.csv --grader-file rubric=examples/audit/rubric.csv --threshold 0.75 --risk-target 0.4
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
The sample accepts three tickets. The reference grader counts one error,
|
|
85
|
+
and the rubric grader counts two. Use `--json` for the changed
|
|
86
|
+
decision, discordant IDs, and per-group counts. For each alternate grader,
|
|
87
|
+
`overall_risk_change` measures the shift on all outputs and
|
|
88
|
+
`excess_risk_change` subtracts that shift from the accepted-set shift.
|
|
89
|
+
Add `--bootstrap 2000 --seed 42` for paired 95% intervals at the frozen
|
|
90
|
+
cutoff. The sample exits 1 because the rubric grader exceeds the 40%
|
|
91
|
+
risk target. A policy with zero accepted rows also exits 1 if you supply
|
|
92
|
+
a target. Invalid input returns 2.
|
|
93
|
+
|
|
94
|
+
For separate grader files, include `id`, `output_sha256`, and a 0/1
|
|
95
|
+
`correct` column in each export. The prediction file supplies the same
|
|
96
|
+
IDs and output hashes plus the score. Use `--output-key-column` if your
|
|
97
|
+
pipeline already records another output identity. If the results are
|
|
98
|
+
already aligned in one CSV, repeat `--correct-column NAME` for each grader.
|
|
99
|
+
The first grader is the reference for directional differences.
|
|
100
|
+
|
|
101
|
+
If a prediction CSV contains multiple runs for the same IDs, select one
|
|
102
|
+
with `--prediction-filter COLUMN=VALUE`.
|
|
103
|
+
[Real routing example and CI setup](https://github.com/Daveyuwang/caat/blob/main/docs/guide/ci-audit.md)
|
|
104
|
+
|
|
105
|
+
## Select a threshold on calibration data
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
caat select examples/ifeval_strict.csv --score-column mean_token_logprob --correct-column strict_pass --thresholds -0.20 -0.35 -0.60 --risk-target 0.4 --rule empirical
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
The example selects `-0.35`, accepting 12 of 24 responses with four
|
|
112
|
+
errors on the calibration sample. Supply a fixed list with `--thresholds`
|
|
113
|
+
or use `--design DESIGN.csv` to derive candidates from score deciles on a
|
|
114
|
+
separate file. Repeat `--correct-column` to require the target under every
|
|
115
|
+
grader. The JSON includes per-candidate counts and `selected_threshold`.
|
|
116
|
+
The command returns 0 if it selects a cutoff, 1 if none qualifies, and 2
|
|
117
|
+
for invalid input.
|
|
118
|
+
|
|
119
|
+
For statistical risk control, use `--rule ltt-bonferroni` (default),
|
|
120
|
+
`--rule ltt-holm`, or `--rule ltt-fixed-sequence`. These rules assume
|
|
121
|
+
independent calibration rows from the target population and a candidate
|
|
122
|
+
grid fixed before calibration. Holm's rule tests candidates in order of
|
|
123
|
+
increasing p-value. [Holm's step-down procedure](https://www.jstor.org/stable/4615733)
|
|
124
|
+
`--failure-rate` defaults to 0.05. Fixed-sequence testing proceeds from
|
|
125
|
+
the strictest threshold and stops at the first failure. The empirical rule
|
|
126
|
+
has no statistical guarantee. Apply the selected cutoff unchanged to
|
|
127
|
+
held-out data with `caat evaluate` or `caat audit`.
|
|
128
|
+
|
|
129
|
+
## Score saved responses
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
caat features examples/token_scores.jsonl --output replay/token_scores.csv
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
The input supplies response text and token log probabilities. The CSV
|
|
136
|
+
contains one row per response, with scores you can use as `--score-column`
|
|
137
|
+
values. [JSONL format and available scores](https://github.com/Daveyuwang/caat/blob/main/docs/guide/features.md)
|
|
138
|
+
|
|
139
|
+
Use `caat scores` to compare candidate columns at matched coverage budgets
|
|
140
|
+
and pick a score on the design split before calibrating a threshold.
|
|
141
|
+
[Score comparison guide](https://github.com/Daveyuwang/caat/blob/main/docs/guide/scores.md)
|
|
142
|
+
|
|
143
|
+
Use `caat queue` to order saved predictions for review under a row budget.
|
|
144
|
+
It can cap the number of rows per group and compare error yield with a
|
|
145
|
+
random queue. [Queue guide](https://github.com/Daveyuwang/caat/blob/main/docs/guide/queue.md) ·
|
|
146
|
+
[Course replay](https://github.com/Daveyuwang/caat/blob/main/docs/research/benchmarks.md#review-queues)
|
|
147
|
+
|
|
148
|
+
## CPU benchmarks
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
python scripts/replay_benchmarks.py
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
The command replays the bundled 24-response samples for IFEval, MMLU-Pro, and
|
|
155
|
+
TriviaQA and writes `replay/benchmarks/summary.csv`. The larger HELM comparison
|
|
156
|
+
evaluates sequence probability and perplexity rankings and tests empirical
|
|
157
|
+
threshold selection against Learn then Test.
|
|
158
|
+
[Sources, results, and reproduction](https://github.com/Daveyuwang/caat/blob/main/docs/research/benchmarks.md) ·
|
|
159
|
+
[WildBench and Omni-MATH grader replay](https://github.com/Daveyuwang/caat/blob/main/docs/research/judge-study.md)
|
|
160
|
+
|
|
161
|
+
The local Qwen2.5-7B study compares token scores, self-reports, and sample
|
|
162
|
+
agreement on TriviaQA and MMLU-Pro. It includes a CPU replay and measured
|
|
163
|
+
collection costs from an RTX 5080.
|
|
164
|
+
[Qwen confidence benchmark](https://github.com/Daveyuwang/caat/blob/main/docs/research/black-box-benchmark.md)
|
|
165
|
+
|
|
166
|
+
## Development
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
python -m pip install -e .
|
|
170
|
+
python -m unittest discover -s tests
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
The tests include the bundled benchmark replay. CI runs them on Python 3.10
|
|
174
|
+
through 3.14 on Linux and on 3.10 and 3.14 on Windows, runs the README
|
|
175
|
+
examples, and builds and installs the wheel.
|
|
176
|
+
|
|
177
|
+
[Evaluation method and related work](https://github.com/Daveyuwang/caat/blob/main/docs/research/method.md)
|
|
178
|
+
|
|
179
|
+
`caat plugins` lists built-in and installed components. The optional
|
|
180
|
+
Transformers backend collects model outputs for later scoring and replay.
|
|
181
|
+
[Plugin guide](https://github.com/Daveyuwang/caat/blob/main/docs/guide/plugins.md) ·
|
|
182
|
+
[Local model collection](https://github.com/Daveyuwang/caat/blob/main/docs/guide/collect.md)
|
|
183
|
+
|
|
184
|
+
[Contributing](https://github.com/Daveyuwang/caat/blob/main/CONTRIBUTING.md) ·
|
|
185
|
+
[Roadmap](https://github.com/Daveyuwang/caat/blob/main/docs/ROADMAP.md) ·
|
|
186
|
+
[Changelog](https://github.com/Daveyuwang/caat/blob/main/CHANGELOG.md)
|
|
187
|
+
|
|
188
|
+
## License
|
|
189
|
+
|
|
190
|
+
MIT. See [LICENSE](https://github.com/Daveyuwang/caat/blob/main/LICENSE).
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "caat-annotation"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Score, calibrate, evaluate, and triage saved model predictions"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
|
+
keywords = ["selective prediction", "risk control", "LLM evaluation", "human review", "calibration"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Programming Language :: Python :: 3.14",
|
|
25
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://github.com/Daveyuwang/caat"
|
|
30
|
+
Documentation = "https://github.com/Daveyuwang/caat/blob/main/README.md"
|
|
31
|
+
Issues = "https://github.com/Daveyuwang/caat/issues"
|
|
32
|
+
|
|
33
|
+
[project.scripts]
|
|
34
|
+
caat = "caat.cli:main"
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
dev = ["ruff==0.14.14"]
|
|
38
|
+
models = ["torch>=2.6", "transformers>=4.57,<6", "accelerate>=1.1"]
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.packages.find]
|
|
41
|
+
where = ["src"]
|
|
42
|
+
|
|
43
|
+
[tool.ruff]
|
|
44
|
+
target-version = "py310"
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""Confidence-Aware Annotation Toolkit."""
|
|
2
|
+
|
|
3
|
+
from .audit import audit
|
|
4
|
+
from .evaluate import evaluate
|
|
5
|
+
from .policy import policy
|
|
6
|
+
from .select import select
|
|
7
|
+
from .table import GradedRow, load_csv
|
|
8
|
+
|
|
9
|
+
__all__ = ["GradedRow", "audit", "evaluate", "load_csv", "policy", "select"]
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Input adapters produce graded rows for CAAT's analysis commands."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Callable
|
|
5
|
+
|
|
6
|
+
from .registry import Registry
|
|
7
|
+
from .table import read_csv_rows
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class Adapter:
|
|
12
|
+
read: Callable
|
|
13
|
+
source_format: str
|
|
14
|
+
source_version: str
|
|
15
|
+
fixture: Callable | None = None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def csv_rows(path, options):
|
|
19
|
+
rows, _ = read_csv_rows(path, **options)
|
|
20
|
+
return rows
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
ADAPTERS = Registry("adapters", Adapter, {"csv": Adapter(csv_rows, "CSV", "1")})
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
from math import ceil, isfinite
|
|
2
|
+
from random import Random
|
|
3
|
+
from typing import Iterable
|
|
4
|
+
|
|
5
|
+
from .metrics import grouped_counts
|
|
6
|
+
from .table import GradedRow, validated_rows
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def percentile_interval(values):
|
|
10
|
+
if not values:
|
|
11
|
+
return None
|
|
12
|
+
ordered = sorted(values)
|
|
13
|
+
return [ordered[ceil(fraction * len(ordered)) - 1] for fraction in (0.025, 0.975)]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def bootstrap_shifts(pairs, replicates, seed):
|
|
17
|
+
# Resample the full rows so the accepted and whole-set risks remain paired.
|
|
18
|
+
rng = Random(seed)
|
|
19
|
+
risk_deltas, whole_deltas, excess_deltas = [], [], []
|
|
20
|
+
for _ in range(replicates):
|
|
21
|
+
accepted = accepted_delta = total_delta = 0
|
|
22
|
+
for kept, change in rng.choices(pairs, k=len(pairs)):
|
|
23
|
+
total_delta += change
|
|
24
|
+
if kept:
|
|
25
|
+
accepted += 1
|
|
26
|
+
accepted_delta += change
|
|
27
|
+
whole = total_delta / len(pairs)
|
|
28
|
+
whole_deltas.append(whole)
|
|
29
|
+
if accepted:
|
|
30
|
+
risk = accepted_delta / accepted
|
|
31
|
+
risk_deltas.append(risk)
|
|
32
|
+
excess_deltas.append(risk - whole)
|
|
33
|
+
return {
|
|
34
|
+
"risk_change": percentile_interval(risk_deltas),
|
|
35
|
+
"overall_risk_change": percentile_interval(whole_deltas),
|
|
36
|
+
"excess_risk_change": percentile_interval(excess_deltas),
|
|
37
|
+
"replicates_with_acceptance": len(risk_deltas),
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def audit(
|
|
42
|
+
rows: Iterable[GradedRow],
|
|
43
|
+
*,
|
|
44
|
+
graders: tuple[str, ...],
|
|
45
|
+
threshold: float,
|
|
46
|
+
risk_target: float | None = None,
|
|
47
|
+
bootstrap: int | None = None,
|
|
48
|
+
seed: int = 42,
|
|
49
|
+
) -> dict:
|
|
50
|
+
"""Replay one cutoff against paired correctness outcomes on the same outputs."""
|
|
51
|
+
if not isfinite(threshold):
|
|
52
|
+
raise ValueError("threshold must be finite")
|
|
53
|
+
if risk_target is not None and not 0 <= risk_target <= 1:
|
|
54
|
+
raise ValueError("risk target must be between 0 and 1")
|
|
55
|
+
if bootstrap is not None and bootstrap < 1:
|
|
56
|
+
raise ValueError("bootstrap count must be positive")
|
|
57
|
+
rows = validated_rows(rows, graders, minimum_graders=2)
|
|
58
|
+
|
|
59
|
+
accepted = [row for row in rows if row.score >= threshold]
|
|
60
|
+
results = {}
|
|
61
|
+
for index, name in enumerate(graders):
|
|
62
|
+
overall, groups = grouped_counts(
|
|
63
|
+
(row.group, row.score >= threshold, not row.outcomes[index]) for row in rows
|
|
64
|
+
)
|
|
65
|
+
risk = overall.error_rate
|
|
66
|
+
results[name] = {
|
|
67
|
+
**overall.metrics(),
|
|
68
|
+
"meets_target": risk <= risk_target
|
|
69
|
+
if risk is not None and risk_target is not None
|
|
70
|
+
else None,
|
|
71
|
+
"by_group": {group: counts.metrics() for group, counts in groups.items()},
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
comparisons = {}
|
|
75
|
+
for index, name in enumerate(graders[1:], start=1):
|
|
76
|
+
newly_failed = sum(
|
|
77
|
+
row.outcomes[0] and not row.outcomes[index] for row in accepted
|
|
78
|
+
)
|
|
79
|
+
newly_passed = sum(
|
|
80
|
+
not row.outcomes[0] and row.outcomes[index] for row in accepted
|
|
81
|
+
)
|
|
82
|
+
primary = results[graders[0]]["meets_target"]
|
|
83
|
+
alternate = results[name]["meets_target"]
|
|
84
|
+
risk_change = (
|
|
85
|
+
(newly_failed - newly_passed) / len(accepted) if accepted else None
|
|
86
|
+
)
|
|
87
|
+
changes = [
|
|
88
|
+
(row.score >= threshold, row.outcomes[0] - row.outcomes[index])
|
|
89
|
+
for row in rows
|
|
90
|
+
]
|
|
91
|
+
overall_change = sum(delta for _, delta in changes) / len(rows)
|
|
92
|
+
comparisons[name] = {
|
|
93
|
+
"newly_failed": newly_failed,
|
|
94
|
+
"newly_passed": newly_passed,
|
|
95
|
+
"risk_change": risk_change,
|
|
96
|
+
"overall_risk_change": overall_change,
|
|
97
|
+
"excess_risk_change": risk_change - overall_change
|
|
98
|
+
if risk_change is not None
|
|
99
|
+
else None,
|
|
100
|
+
"target_decision_changed": primary != alternate
|
|
101
|
+
if primary is not None
|
|
102
|
+
else None,
|
|
103
|
+
}
|
|
104
|
+
if bootstrap is not None:
|
|
105
|
+
comparisons[name]["bootstrap_95"] = bootstrap_shifts(
|
|
106
|
+
changes, bootstrap, seed
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
discordant = [
|
|
110
|
+
row for row in rows if any(bit != row.outcomes[0] for bit in row.outcomes[1:])
|
|
111
|
+
]
|
|
112
|
+
report = {
|
|
113
|
+
"schema_version": 1,
|
|
114
|
+
"threshold": threshold,
|
|
115
|
+
"risk_target": risk_target,
|
|
116
|
+
"primary_grader": graders[0],
|
|
117
|
+
"graders": results,
|
|
118
|
+
"accepted_comparisons": comparisons,
|
|
119
|
+
"discordant": [
|
|
120
|
+
{
|
|
121
|
+
"id": row.id,
|
|
122
|
+
"group": row.group,
|
|
123
|
+
"accepted": row.score >= threshold,
|
|
124
|
+
"outcomes": dict(zip(graders, row.outcomes, strict=True)),
|
|
125
|
+
}
|
|
126
|
+
for row in sorted(discordant, key=lambda row: row.id)
|
|
127
|
+
],
|
|
128
|
+
}
|
|
129
|
+
if bootstrap is not None:
|
|
130
|
+
report["bootstrap"] = {"replicates": bootstrap, "seed": seed}
|
|
131
|
+
return report
|