caat-annotation 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. caat_annotation-0.2.0/LICENSE +21 -0
  2. caat_annotation-0.2.0/PKG-INFO +221 -0
  3. caat_annotation-0.2.0/README.md +190 -0
  4. caat_annotation-0.2.0/pyproject.toml +44 -0
  5. caat_annotation-0.2.0/setup.cfg +4 -0
  6. caat_annotation-0.2.0/src/caat/__init__.py +9 -0
  7. caat_annotation-0.2.0/src/caat/adapters.py +23 -0
  8. caat_annotation-0.2.0/src/caat/audit.py +131 -0
  9. caat_annotation-0.2.0/src/caat/calibrate.py +328 -0
  10. caat_annotation-0.2.0/src/caat/cli.py +761 -0
  11. caat_annotation-0.2.0/src/caat/collect.py +205 -0
  12. caat_annotation-0.2.0/src/caat/evaluate.py +66 -0
  13. caat_annotation-0.2.0/src/caat/features.py +304 -0
  14. caat_annotation-0.2.0/src/caat/html_report.py +130 -0
  15. caat_annotation-0.2.0/src/caat/metrics.py +196 -0
  16. caat_annotation-0.2.0/src/caat/plots.py +223 -0
  17. caat_annotation-0.2.0/src/caat/plugins.py +32 -0
  18. caat_annotation-0.2.0/src/caat/policy.py +127 -0
  19. caat_annotation-0.2.0/src/caat/queue.py +75 -0
  20. caat_annotation-0.2.0/src/caat/registry.py +71 -0
  21. caat_annotation-0.2.0/src/caat/renderers.py +64 -0
  22. caat_annotation-0.2.0/src/caat/report.py +209 -0
  23. caat_annotation-0.2.0/src/caat/rules.py +127 -0
  24. caat_annotation-0.2.0/src/caat/scores.py +231 -0
  25. caat_annotation-0.2.0/src/caat/select.py +88 -0
  26. caat_annotation-0.2.0/src/caat/table.py +236 -0
  27. caat_annotation-0.2.0/src/caat_annotation.egg-info/PKG-INFO +221 -0
  28. caat_annotation-0.2.0/src/caat_annotation.egg-info/SOURCES.txt +40 -0
  29. caat_annotation-0.2.0/src/caat_annotation.egg-info/dependency_links.txt +1 -0
  30. caat_annotation-0.2.0/src/caat_annotation.egg-info/entry_points.txt +2 -0
  31. caat_annotation-0.2.0/src/caat_annotation.egg-info/requires.txt +8 -0
  32. caat_annotation-0.2.0/src/caat_annotation.egg-info/top_level.txt +1 -0
  33. caat_annotation-0.2.0/tests/test_caat_audit.py +215 -0
  34. caat_annotation-0.2.0/tests/test_caat_calibration.py +143 -0
  35. caat_annotation-0.2.0/tests/test_caat_evaluate.py +113 -0
  36. caat_annotation-0.2.0/tests/test_caat_features.py +92 -0
  37. caat_annotation-0.2.0/tests/test_caat_policy.py +131 -0
  38. caat_annotation-0.2.0/tests/test_caat_scores.py +75 -0
  39. caat_annotation-0.2.0/tests/test_caat_selection.py +132 -0
  40. caat_annotation-0.2.0/tests/test_caat_use.py +133 -0
  41. caat_annotation-0.2.0/tests/test_readme_reports.py +223 -0
  42. caat_annotation-0.2.0/tests/test_replay_benchmarks.py +57 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025-2026 The CAAT contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,221 @@
1
+ Metadata-Version: 2.4
2
+ Name: caat-annotation
3
+ Version: 0.2.0
4
+ Summary: Score, calibrate, evaluate, and triage saved model predictions
5
+ License-Expression: MIT
6
+ Project-URL: Homepage, https://github.com/Daveyuwang/caat
7
+ Project-URL: Documentation, https://github.com/Daveyuwang/caat/blob/main/README.md
8
+ Project-URL: Issues, https://github.com/Daveyuwang/caat/issues
9
+ Keywords: selective prediction,risk control,LLM evaluation,human review,calibration
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Environment :: Console
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Programming Language :: Python :: 3 :: Only
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Programming Language :: Python :: 3.14
20
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
21
+ Requires-Python: >=3.10
22
+ Description-Content-Type: text/markdown
23
+ License-File: LICENSE
24
+ Provides-Extra: dev
25
+ Requires-Dist: ruff==0.14.14; extra == "dev"
26
+ Provides-Extra: models
27
+ Requires-Dist: torch>=2.6; extra == "models"
28
+ Requires-Dist: transformers<6,>=4.57; extra == "models"
29
+ Requires-Dist: accelerate>=1.1; extra == "models"
30
+ Dynamic: license-file
31
+
32
+ # Confidence-Aware Annotation Toolkit
33
+
34
+ [![CI](https://github.com/Daveyuwang/caat/actions/workflows/ci.yml/badge.svg)](https://github.com/Daveyuwang/caat/actions/workflows/ci.yml)
35
+
36
+ CAAT scores, calibrates, evaluates, and triages saved model predictions.
37
+ Compare confidence signals, measure correctness probabilities, and choose
38
+ which outputs to accept automatically or send for review. The analysis
39
+ commands run on a CPU with Python 3.10 or later.
40
+
41
+ ```bash
42
+ git clone https://github.com/Daveyuwang/caat.git
43
+ cd caat
44
+ python -m pip install .
45
+ caat evaluate examples/ticket_triage.csv --id-column ticket_id --prediction-column model_route --reference-column expected_route --score-column confidence --group-column inbox --threshold 0.75 --probability --output-dir replay/tickets
46
+ ```
47
+
48
+ The example accepts 3 of 5 tickets and makes 1 error. CAAT writes `metrics.json`
49
+ with overall and per-group counts, AUROC, area under the risk–coverage curve
50
+ (AURC), and normalized AURC gain. It writes `risk_coverage.csv` with the
51
+ results at each distinct score threshold. A normalized gain of 0 matches
52
+ random ordering and 1 matches ordering all correct predictions first.
53
+
54
+ ## CSV input
55
+
56
+ Each row needs a unique `id` and a finite `score`. Larger scores indicate
57
+ greater confidence. Supply `prediction` and `reference` for exact label
58
+ matching, or use `--correct-column NAME` for a grader's 0/1 outcome. Map
59
+ different headers with the flags above. An optional `group` column gives
60
+ per-group metrics.
61
+
62
+ CAAT accepts every row with a score at or above `--threshold`. Use
63
+ `--probability` to report Brier score, ECE, ACE, and reliability bins for
64
+ probabilities of correctness.
65
+ The review commands print a short summary. Add `--json` for machine-readable
66
+ stdout or `--output FILE` to save the JSON report. The CLI and Python API
67
+ use the standard library. [CSV input](https://github.com/Daveyuwang/caat/blob/main/docs/guide/csv-format.md) · [Report fields](https://github.com/Daveyuwang/caat/blob/main/docs/guide/reports.md)
68
+
69
+ Use `caat calibrate fit` to save a temperature, Platt, isotonic, or histogram
70
+ map from labeled calibration rows. Use `caat calibrate apply` to add its
71
+ probabilities to a held-out CSV. Add `--html report.html` to `evaluate`,
72
+ `audit`, or `policy` for a self-contained report.
73
+ [Calibration examples](https://github.com/Daveyuwang/caat/blob/main/docs/guide/calibration.md)
74
+
75
+ ![Reliability diagram for the ticket-triage example](https://github.com/Daveyuwang/caat/raw/main/docs/figures/calibration-example.svg)
76
+
77
+ ## Choose and check a cutoff in one command
78
+
79
+ ```bash
80
+ caat policy examples/ifeval_graders.csv --score-column mean_token_logprob --correct-column strict_pass --check-column helm_strict_pass --risk-target 0.4 --plot replay/policy.svg
81
+ ```
82
+
83
+ `caat policy` splits one labeled CSV into design, calibration, and held-out
84
+ rows. It selects a cutoff on the calibration rows and applies it unchanged
85
+ to the held-out rows. The example holds 541 saved IFEval responses graded by
86
+ Google's strict grader and by HELM. The command selects
87
+ `-0.43964836263206303` under the Google grader. On the 217 held-out rows the
88
+ policy accepts 144 responses, with 33 errors (22.9%) under Google and 45
89
+ (31.2%) under HELM. Both meet the 40% target.
90
+
91
+ ![Held-out IFEval risk for the Google and HELM graders](https://github.com/Daveyuwang/caat/raw/main/docs/figures/policy-example.svg)
92
+
93
+ A `--check-column` grader is reported on held-out rows and takes no part in
94
+ selection. Pass it as a second `--correct-column` to require the target
95
+ under both graders. The cutoff then rises to `-0.3327795407672605` and the
96
+ policy accepts 92 held-out responses, with 19 and 23 errors.
97
+
98
+ The summary reports AUROC for each grader on the calibration rows and says
99
+ when the score ranks no better than random. The split sorts IDs, shuffles
100
+ with `--seed` (default 42), and assigns about 20%, 40%, and 40% of the rows.
101
+ `--rule` and `--failure-rate` work as in `caat select`. Add `--json` for the
102
+ full report. The command returns 0 if the held-out rows meet the target
103
+ under every listed grader, 1 if no cutoff qualifies or a grader exceeds the
104
+ target, and 2 for invalid input.
105
+
106
+ ## Compare graders on the same outputs
107
+
108
+ Run a frozen threshold against separate grader exports. CAAT matches rows
109
+ by ID and checks that each grader scored the same saved output.
110
+
111
+ ```bash
112
+ caat audit examples/audit/predictions.csv --output-key-column output_key --grader-file reference=examples/audit/reference.csv --grader-file rubric=examples/audit/rubric.csv --threshold 0.75 --risk-target 0.4
113
+ ```
114
+
115
+ The sample accepts three tickets. The reference grader counts one error,
116
+ and the rubric grader counts two. Use `--json` for the changed
117
+ decision, discordant IDs, and per-group counts. For each alternate grader,
118
+ `overall_risk_change` measures the shift on all outputs and
119
+ `excess_risk_change` subtracts that shift from the accepted-set shift.
120
+ Add `--bootstrap 2000 --seed 42` for paired 95% intervals at the frozen
121
+ cutoff. The sample exits 1 because the rubric grader exceeds the 40%
122
+ risk target. A policy with zero accepted rows also exits 1 if you supply
123
+ a target. Invalid input returns 2.
124
+
125
+ For separate grader files, include `id`, `output_sha256`, and a 0/1
126
+ `correct` column in each export. The prediction file supplies the same
127
+ IDs and output hashes plus the score. Use `--output-key-column` if your
128
+ pipeline already records another output identity. If the results are
129
+ already aligned in one CSV, repeat `--correct-column NAME` for each grader.
130
+ The first grader is the reference for directional differences.
131
+
132
+ If a prediction CSV contains multiple runs for the same IDs, select one
133
+ with `--prediction-filter COLUMN=VALUE`.
134
+ [Real routing example and CI setup](https://github.com/Daveyuwang/caat/blob/main/docs/guide/ci-audit.md)
135
+
136
+ ## Select a threshold on calibration data
137
+
138
+ ```bash
139
+ caat select examples/ifeval_strict.csv --score-column mean_token_logprob --correct-column strict_pass --thresholds -0.20 -0.35 -0.60 --risk-target 0.4 --rule empirical
140
+ ```
141
+
142
+ The example selects `-0.35`, accepting 12 of 24 responses with four
143
+ errors on the calibration sample. Supply a fixed list with `--thresholds`
144
+ or use `--design DESIGN.csv` to derive candidates from score deciles on a
145
+ separate file. Repeat `--correct-column` to require the target under every
146
+ grader. The JSON includes per-candidate counts and `selected_threshold`.
147
+ The command returns 0 if it selects a cutoff, 1 if none qualifies, and 2
148
+ for invalid input.
149
+
150
+ For statistical risk control, use `--rule ltt-bonferroni` (default),
151
+ `--rule ltt-holm`, or `--rule ltt-fixed-sequence`. These rules assume
152
+ independent calibration rows from the target population and a candidate
153
+ grid fixed before calibration. Holm's rule tests candidates in order of
154
+ increasing p-value. [Holm's step-down procedure](https://www.jstor.org/stable/4615733)
155
+ `--failure-rate` defaults to 0.05. Fixed-sequence testing proceeds from
156
+ the strictest threshold and stops at the first failure. The empirical rule
157
+ has no statistical guarantee. Apply the selected cutoff unchanged to
158
+ held-out data with `caat evaluate` or `caat audit`.
159
+
160
+ ## Score saved responses
161
+
162
+ ```bash
163
+ caat features examples/token_scores.jsonl --output replay/token_scores.csv
164
+ ```
165
+
166
+ The input supplies response text and token log probabilities. The CSV
167
+ contains one row per response, with scores you can use as `--score-column`
168
+ values. [JSONL format and available scores](https://github.com/Daveyuwang/caat/blob/main/docs/guide/features.md)
169
+
170
+ Use `caat scores` to compare candidate columns at matched coverage budgets
171
+ and pick a score on the design split before calibrating a threshold.
172
+ [Score comparison guide](https://github.com/Daveyuwang/caat/blob/main/docs/guide/scores.md)
173
+
174
+ Use `caat queue` to order saved predictions for review under a row budget.
175
+ It can cap the number of rows per group and compare error yield with a
176
+ random queue. [Queue guide](https://github.com/Daveyuwang/caat/blob/main/docs/guide/queue.md) ·
177
+ [Course replay](https://github.com/Daveyuwang/caat/blob/main/docs/research/benchmarks.md#review-queues)
178
+
179
+ ## CPU benchmarks
180
+
181
+ ```bash
182
+ python scripts/replay_benchmarks.py
183
+ ```
184
+
185
+ The command replays the bundled 24-response samples for IFEval, MMLU-Pro, and
186
+ TriviaQA and writes `replay/benchmarks/summary.csv`. The larger HELM comparison
187
+ evaluates sequence probability and perplexity rankings and tests empirical
188
+ threshold selection against Learn then Test.
189
+ [Sources, results, and reproduction](https://github.com/Daveyuwang/caat/blob/main/docs/research/benchmarks.md) ·
190
+ [WildBench and Omni-MATH grader replay](https://github.com/Daveyuwang/caat/blob/main/docs/research/judge-study.md)
191
+
192
+ The local Qwen2.5-7B study compares token scores, self-reports, and sample
193
+ agreement on TriviaQA and MMLU-Pro. It includes a CPU replay and measured
194
+ collection costs from an RTX 5080.
195
+ [Qwen confidence benchmark](https://github.com/Daveyuwang/caat/blob/main/docs/research/black-box-benchmark.md)
196
+
197
+ ## Development
198
+
199
+ ```bash
200
+ python -m pip install -e .
201
+ python -m unittest discover -s tests
202
+ ```
203
+
204
+ The tests include the bundled benchmark replay. CI runs them on Python 3.10
205
+ through 3.14 on Linux and on 3.10 and 3.14 on Windows, runs the README
206
+ examples, and builds and installs the wheel.
207
+
208
+ [Evaluation method and related work](https://github.com/Daveyuwang/caat/blob/main/docs/research/method.md)
209
+
210
+ `caat plugins` lists built-in and installed components. The optional
211
+ Transformers backend collects model outputs for later scoring and replay.
212
+ [Plugin guide](https://github.com/Daveyuwang/caat/blob/main/docs/guide/plugins.md) ·
213
+ [Local model collection](https://github.com/Daveyuwang/caat/blob/main/docs/guide/collect.md)
214
+
215
+ [Contributing](https://github.com/Daveyuwang/caat/blob/main/CONTRIBUTING.md) ·
216
+ [Roadmap](https://github.com/Daveyuwang/caat/blob/main/docs/ROADMAP.md) ·
217
+ [Changelog](https://github.com/Daveyuwang/caat/blob/main/CHANGELOG.md)
218
+
219
+ ## License
220
+
221
+ MIT. See [LICENSE](https://github.com/Daveyuwang/caat/blob/main/LICENSE).
@@ -0,0 +1,190 @@
1
+ # Confidence-Aware Annotation Toolkit
2
+
3
+ [![CI](https://github.com/Daveyuwang/caat/actions/workflows/ci.yml/badge.svg)](https://github.com/Daveyuwang/caat/actions/workflows/ci.yml)
4
+
5
+ CAAT scores, calibrates, evaluates, and triages saved model predictions.
6
+ Compare confidence signals, measure correctness probabilities, and choose
7
+ which outputs to accept automatically or send for review. The analysis
8
+ commands run on a CPU with Python 3.10 or later.
9
+
10
+ ```bash
11
+ git clone https://github.com/Daveyuwang/caat.git
12
+ cd caat
13
+ python -m pip install .
14
+ caat evaluate examples/ticket_triage.csv --id-column ticket_id --prediction-column model_route --reference-column expected_route --score-column confidence --group-column inbox --threshold 0.75 --probability --output-dir replay/tickets
15
+ ```
16
+
17
+ The example accepts 3 of 5 tickets and makes 1 error. CAAT writes `metrics.json`
18
+ with overall and per-group counts, AUROC, area under the risk–coverage curve
19
+ (AURC), and normalized AURC gain. It writes `risk_coverage.csv` with the
20
+ results at each distinct score threshold. A normalized gain of 0 matches
21
+ random ordering and 1 matches ordering all correct predictions first.
22
+
23
+ ## CSV input
24
+
25
+ Each row needs a unique `id` and a finite `score`. Larger scores indicate
26
+ greater confidence. Supply `prediction` and `reference` for exact label
27
+ matching, or use `--correct-column NAME` for a grader's 0/1 outcome. Map
28
+ different headers with the flags above. An optional `group` column gives
29
+ per-group metrics.
30
+
31
+ CAAT accepts every row with a score at or above `--threshold`. Use
32
+ `--probability` to report Brier score, ECE, ACE, and reliability bins for
33
+ probabilities of correctness.
34
+ The review commands print a short summary. Add `--json` for machine-readable
35
+ stdout or `--output FILE` to save the JSON report. The CLI and Python API
36
+ use the standard library. [CSV input](https://github.com/Daveyuwang/caat/blob/main/docs/guide/csv-format.md) · [Report fields](https://github.com/Daveyuwang/caat/blob/main/docs/guide/reports.md)
37
+
38
+ Use `caat calibrate fit` to save a temperature, Platt, isotonic, or histogram
39
+ map from labeled calibration rows. Use `caat calibrate apply` to add its
40
+ probabilities to a held-out CSV. Add `--html report.html` to `evaluate`,
41
+ `audit`, or `policy` for a self-contained report.
42
+ [Calibration examples](https://github.com/Daveyuwang/caat/blob/main/docs/guide/calibration.md)
43
+
44
+ ![Reliability diagram for the ticket-triage example](https://github.com/Daveyuwang/caat/raw/main/docs/figures/calibration-example.svg)
45
+
46
+ ## Choose and check a cutoff in one command
47
+
48
+ ```bash
49
+ caat policy examples/ifeval_graders.csv --score-column mean_token_logprob --correct-column strict_pass --check-column helm_strict_pass --risk-target 0.4 --plot replay/policy.svg
50
+ ```
51
+
52
+ `caat policy` splits one labeled CSV into design, calibration, and held-out
53
+ rows. It selects a cutoff on the calibration rows and applies it unchanged
54
+ to the held-out rows. The example holds 541 saved IFEval responses graded by
55
+ Google's strict grader and by HELM. The command selects
56
+ `-0.43964836263206303` under the Google grader. On the 217 held-out rows the
57
+ policy accepts 144 responses, with 33 errors (22.9%) under Google and 45
58
+ (31.2%) under HELM. Both meet the 40% target.
59
+
60
+ ![Held-out IFEval risk for the Google and HELM graders](https://github.com/Daveyuwang/caat/raw/main/docs/figures/policy-example.svg)
61
+
62
+ A `--check-column` grader is reported on held-out rows and takes no part in
63
+ selection. Pass it as a second `--correct-column` to require the target
64
+ under both graders. The cutoff then rises to `-0.3327795407672605` and the
65
+ policy accepts 92 held-out responses, with 19 and 23 errors.
66
+
67
+ The summary reports AUROC for each grader on the calibration rows and says
68
+ when the score ranks no better than random. The split sorts IDs, shuffles
69
+ with `--seed` (default 42), and assigns about 20%, 40%, and 40% of the rows.
70
+ `--rule` and `--failure-rate` work as in `caat select`. Add `--json` for the
71
+ full report. The command returns 0 if the held-out rows meet the target
72
+ under every listed grader, 1 if no cutoff qualifies or a grader exceeds the
73
+ target, and 2 for invalid input.
74
+
75
+ ## Compare graders on the same outputs
76
+
77
+ Run a frozen threshold against separate grader exports. CAAT matches rows
78
+ by ID and checks that each grader scored the same saved output.
79
+
80
+ ```bash
81
+ caat audit examples/audit/predictions.csv --output-key-column output_key --grader-file reference=examples/audit/reference.csv --grader-file rubric=examples/audit/rubric.csv --threshold 0.75 --risk-target 0.4
82
+ ```
83
+
84
+ The sample accepts three tickets. The reference grader counts one error,
85
+ and the rubric grader counts two. Use `--json` for the changed
86
+ decision, discordant IDs, and per-group counts. For each alternate grader,
87
+ `overall_risk_change` measures the shift on all outputs and
88
+ `excess_risk_change` subtracts that shift from the accepted-set shift.
89
+ Add `--bootstrap 2000 --seed 42` for paired 95% intervals at the frozen
90
+ cutoff. The sample exits 1 because the rubric grader exceeds the 40%
91
+ risk target. A policy with zero accepted rows also exits 1 if you supply
92
+ a target. Invalid input returns 2.
93
+
94
+ For separate grader files, include `id`, `output_sha256`, and a 0/1
95
+ `correct` column in each export. The prediction file supplies the same
96
+ IDs and output hashes plus the score. Use `--output-key-column` if your
97
+ pipeline already records another output identity. If the results are
98
+ already aligned in one CSV, repeat `--correct-column NAME` for each grader.
99
+ The first grader is the reference for directional differences.
100
+
101
+ If a prediction CSV contains multiple runs for the same IDs, select one
102
+ with `--prediction-filter COLUMN=VALUE`.
103
+ [Real routing example and CI setup](https://github.com/Daveyuwang/caat/blob/main/docs/guide/ci-audit.md)
104
+
105
+ ## Select a threshold on calibration data
106
+
107
+ ```bash
108
+ caat select examples/ifeval_strict.csv --score-column mean_token_logprob --correct-column strict_pass --thresholds -0.20 -0.35 -0.60 --risk-target 0.4 --rule empirical
109
+ ```
110
+
111
+ The example selects `-0.35`, accepting 12 of 24 responses with four
112
+ errors on the calibration sample. Supply a fixed list with `--thresholds`
113
+ or use `--design DESIGN.csv` to derive candidates from score deciles on a
114
+ separate file. Repeat `--correct-column` to require the target under every
115
+ grader. The JSON includes per-candidate counts and `selected_threshold`.
116
+ The command returns 0 if it selects a cutoff, 1 if none qualifies, and 2
117
+ for invalid input.
118
+
119
+ For statistical risk control, use `--rule ltt-bonferroni` (default),
120
+ `--rule ltt-holm`, or `--rule ltt-fixed-sequence`. These rules assume
121
+ independent calibration rows from the target population and a candidate
122
+ grid fixed before calibration. Holm's rule tests candidates in order of
123
+ increasing p-value. [Holm's step-down procedure](https://www.jstor.org/stable/4615733)
124
+ `--failure-rate` defaults to 0.05. Fixed-sequence testing proceeds from
125
+ the strictest threshold and stops at the first failure. The empirical rule
126
+ has no statistical guarantee. Apply the selected cutoff unchanged to
127
+ held-out data with `caat evaluate` or `caat audit`.
128
+
129
+ ## Score saved responses
130
+
131
+ ```bash
132
+ caat features examples/token_scores.jsonl --output replay/token_scores.csv
133
+ ```
134
+
135
+ The input supplies response text and token log probabilities. The CSV
136
+ contains one row per response, with scores you can use as `--score-column`
137
+ values. [JSONL format and available scores](https://github.com/Daveyuwang/caat/blob/main/docs/guide/features.md)
138
+
139
+ Use `caat scores` to compare candidate columns at matched coverage budgets
140
+ and pick a score on the design split before calibrating a threshold.
141
+ [Score comparison guide](https://github.com/Daveyuwang/caat/blob/main/docs/guide/scores.md)
142
+
143
+ Use `caat queue` to order saved predictions for review under a row budget.
144
+ It can cap the number of rows per group and compare error yield with a
145
+ random queue. [Queue guide](https://github.com/Daveyuwang/caat/blob/main/docs/guide/queue.md) ·
146
+ [Course replay](https://github.com/Daveyuwang/caat/blob/main/docs/research/benchmarks.md#review-queues)
147
+
148
+ ## CPU benchmarks
149
+
150
+ ```bash
151
+ python scripts/replay_benchmarks.py
152
+ ```
153
+
154
+ The command replays the bundled 24-response samples for IFEval, MMLU-Pro, and
155
+ TriviaQA and writes `replay/benchmarks/summary.csv`. The larger HELM comparison
156
+ evaluates sequence probability and perplexity rankings and tests empirical
157
+ threshold selection against Learn then Test.
158
+ [Sources, results, and reproduction](https://github.com/Daveyuwang/caat/blob/main/docs/research/benchmarks.md) ·
159
+ [WildBench and Omni-MATH grader replay](https://github.com/Daveyuwang/caat/blob/main/docs/research/judge-study.md)
160
+
161
+ The local Qwen2.5-7B study compares token scores, self-reports, and sample
162
+ agreement on TriviaQA and MMLU-Pro. It includes a CPU replay and measured
163
+ collection costs from an RTX 5080.
164
+ [Qwen confidence benchmark](https://github.com/Daveyuwang/caat/blob/main/docs/research/black-box-benchmark.md)
165
+
166
+ ## Development
167
+
168
+ ```bash
169
+ python -m pip install -e .
170
+ python -m unittest discover -s tests
171
+ ```
172
+
173
+ The tests include the bundled benchmark replay. CI runs them on Python 3.10
174
+ through 3.14 on Linux and on 3.10 and 3.14 on Windows, runs the README
175
+ examples, and builds and installs the wheel.
176
+
177
+ [Evaluation method and related work](https://github.com/Daveyuwang/caat/blob/main/docs/research/method.md)
178
+
179
+ `caat plugins` lists built-in and installed components. The optional
180
+ Transformers backend collects model outputs for later scoring and replay.
181
+ [Plugin guide](https://github.com/Daveyuwang/caat/blob/main/docs/guide/plugins.md) ·
182
+ [Local model collection](https://github.com/Daveyuwang/caat/blob/main/docs/guide/collect.md)
183
+
184
+ [Contributing](https://github.com/Daveyuwang/caat/blob/main/CONTRIBUTING.md) ·
185
+ [Roadmap](https://github.com/Daveyuwang/caat/blob/main/docs/ROADMAP.md) ·
186
+ [Changelog](https://github.com/Daveyuwang/caat/blob/main/CHANGELOG.md)
187
+
188
+ ## License
189
+
190
+ MIT. See [LICENSE](https://github.com/Daveyuwang/caat/blob/main/LICENSE).
@@ -0,0 +1,44 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "caat-annotation"
7
+ version = "0.2.0"
8
+ description = "Score, calibrate, evaluate, and triage saved model predictions"
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ requires-python = ">=3.10"
13
+ keywords = ["selective prediction", "risk control", "LLM evaluation", "human review", "calibration"]
14
+ classifiers = [
15
+ "Development Status :: 3 - Alpha",
16
+ "Environment :: Console",
17
+ "Intended Audience :: Science/Research",
18
+ "Intended Audience :: Developers",
19
+ "Programming Language :: Python :: 3 :: Only",
20
+ "Programming Language :: Python :: 3.10",
21
+ "Programming Language :: Python :: 3.11",
22
+ "Programming Language :: Python :: 3.12",
23
+ "Programming Language :: Python :: 3.13",
24
+ "Programming Language :: Python :: 3.14",
25
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
26
+ ]
27
+
28
+ [project.urls]
29
+ Homepage = "https://github.com/Daveyuwang/caat"
30
+ Documentation = "https://github.com/Daveyuwang/caat/blob/main/README.md"
31
+ Issues = "https://github.com/Daveyuwang/caat/issues"
32
+
33
+ [project.scripts]
34
+ caat = "caat.cli:main"
35
+
36
+ [project.optional-dependencies]
37
+ dev = ["ruff==0.14.14"]
38
+ models = ["torch>=2.6", "transformers>=4.57,<6", "accelerate>=1.1"]
39
+
40
+ [tool.setuptools.packages.find]
41
+ where = ["src"]
42
+
43
+ [tool.ruff]
44
+ target-version = "py310"
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,9 @@
1
+ """Confidence-Aware Annotation Toolkit."""
2
+
3
+ from .audit import audit
4
+ from .evaluate import evaluate
5
+ from .policy import policy
6
+ from .select import select
7
+ from .table import GradedRow, load_csv
8
+
9
+ __all__ = ["GradedRow", "audit", "evaluate", "load_csv", "policy", "select"]
@@ -0,0 +1,23 @@
1
+ """Input adapters produce graded rows for CAAT's analysis commands."""
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Callable
5
+
6
+ from .registry import Registry
7
+ from .table import read_csv_rows
8
+
9
+
10
+ @dataclass(frozen=True)
11
+ class Adapter:
12
+ read: Callable
13
+ source_format: str
14
+ source_version: str
15
+ fixture: Callable | None = None
16
+
17
+
18
+ def csv_rows(path, options):
19
+ rows, _ = read_csv_rows(path, **options)
20
+ return rows
21
+
22
+
23
+ ADAPTERS = Registry("adapters", Adapter, {"csv": Adapter(csv_rows, "CSV", "1")})
@@ -0,0 +1,131 @@
1
+ from math import ceil, isfinite
2
+ from random import Random
3
+ from typing import Iterable
4
+
5
+ from .metrics import grouped_counts
6
+ from .table import GradedRow, validated_rows
7
+
8
+
9
+ def percentile_interval(values):
10
+ if not values:
11
+ return None
12
+ ordered = sorted(values)
13
+ return [ordered[ceil(fraction * len(ordered)) - 1] for fraction in (0.025, 0.975)]
14
+
15
+
16
+ def bootstrap_shifts(pairs, replicates, seed):
17
+ # Resample the full rows so the accepted and whole-set risks remain paired.
18
+ rng = Random(seed)
19
+ risk_deltas, whole_deltas, excess_deltas = [], [], []
20
+ for _ in range(replicates):
21
+ accepted = accepted_delta = total_delta = 0
22
+ for kept, change in rng.choices(pairs, k=len(pairs)):
23
+ total_delta += change
24
+ if kept:
25
+ accepted += 1
26
+ accepted_delta += change
27
+ whole = total_delta / len(pairs)
28
+ whole_deltas.append(whole)
29
+ if accepted:
30
+ risk = accepted_delta / accepted
31
+ risk_deltas.append(risk)
32
+ excess_deltas.append(risk - whole)
33
+ return {
34
+ "risk_change": percentile_interval(risk_deltas),
35
+ "overall_risk_change": percentile_interval(whole_deltas),
36
+ "excess_risk_change": percentile_interval(excess_deltas),
37
+ "replicates_with_acceptance": len(risk_deltas),
38
+ }
39
+
40
+
41
+ def audit(
42
+ rows: Iterable[GradedRow],
43
+ *,
44
+ graders: tuple[str, ...],
45
+ threshold: float,
46
+ risk_target: float | None = None,
47
+ bootstrap: int | None = None,
48
+ seed: int = 42,
49
+ ) -> dict:
50
+ """Replay one cutoff against paired correctness outcomes on the same outputs."""
51
+ if not isfinite(threshold):
52
+ raise ValueError("threshold must be finite")
53
+ if risk_target is not None and not 0 <= risk_target <= 1:
54
+ raise ValueError("risk target must be between 0 and 1")
55
+ if bootstrap is not None and bootstrap < 1:
56
+ raise ValueError("bootstrap count must be positive")
57
+ rows = validated_rows(rows, graders, minimum_graders=2)
58
+
59
+ accepted = [row for row in rows if row.score >= threshold]
60
+ results = {}
61
+ for index, name in enumerate(graders):
62
+ overall, groups = grouped_counts(
63
+ (row.group, row.score >= threshold, not row.outcomes[index]) for row in rows
64
+ )
65
+ risk = overall.error_rate
66
+ results[name] = {
67
+ **overall.metrics(),
68
+ "meets_target": risk <= risk_target
69
+ if risk is not None and risk_target is not None
70
+ else None,
71
+ "by_group": {group: counts.metrics() for group, counts in groups.items()},
72
+ }
73
+
74
+ comparisons = {}
75
+ for index, name in enumerate(graders[1:], start=1):
76
+ newly_failed = sum(
77
+ row.outcomes[0] and not row.outcomes[index] for row in accepted
78
+ )
79
+ newly_passed = sum(
80
+ not row.outcomes[0] and row.outcomes[index] for row in accepted
81
+ )
82
+ primary = results[graders[0]]["meets_target"]
83
+ alternate = results[name]["meets_target"]
84
+ risk_change = (
85
+ (newly_failed - newly_passed) / len(accepted) if accepted else None
86
+ )
87
+ changes = [
88
+ (row.score >= threshold, row.outcomes[0] - row.outcomes[index])
89
+ for row in rows
90
+ ]
91
+ overall_change = sum(delta for _, delta in changes) / len(rows)
92
+ comparisons[name] = {
93
+ "newly_failed": newly_failed,
94
+ "newly_passed": newly_passed,
95
+ "risk_change": risk_change,
96
+ "overall_risk_change": overall_change,
97
+ "excess_risk_change": risk_change - overall_change
98
+ if risk_change is not None
99
+ else None,
100
+ "target_decision_changed": primary != alternate
101
+ if primary is not None
102
+ else None,
103
+ }
104
+ if bootstrap is not None:
105
+ comparisons[name]["bootstrap_95"] = bootstrap_shifts(
106
+ changes, bootstrap, seed
107
+ )
108
+
109
+ discordant = [
110
+ row for row in rows if any(bit != row.outcomes[0] for bit in row.outcomes[1:])
111
+ ]
112
+ report = {
113
+ "schema_version": 1,
114
+ "threshold": threshold,
115
+ "risk_target": risk_target,
116
+ "primary_grader": graders[0],
117
+ "graders": results,
118
+ "accepted_comparisons": comparisons,
119
+ "discordant": [
120
+ {
121
+ "id": row.id,
122
+ "group": row.group,
123
+ "accepted": row.score >= threshold,
124
+ "outcomes": dict(zip(graders, row.outcomes, strict=True)),
125
+ }
126
+ for row in sorted(discordant, key=lambda row: row.id)
127
+ ],
128
+ }
129
+ if bootstrap is not None:
130
+ report["bootstrap"] = {"replicates": bootstrap, "seed": seed}
131
+ return report