caat-annotation 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
caat/__init__.py ADDED
@@ -0,0 +1,9 @@
1
+ """Confidence-Aware Annotation Toolkit."""
2
+
3
+ from .audit import audit
4
+ from .evaluate import evaluate
5
+ from .policy import policy
6
+ from .select import select
7
+ from .table import GradedRow, load_csv
8
+
9
+ __all__ = ["GradedRow", "audit", "evaluate", "load_csv", "policy", "select"]
caat/adapters.py ADDED
@@ -0,0 +1,23 @@
1
+ """Input adapters produce graded rows for CAAT's analysis commands."""
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Callable
5
+
6
+ from .registry import Registry
7
+ from .table import read_csv_rows
8
+
9
+
10
+ @dataclass(frozen=True)
11
+ class Adapter:
12
+ read: Callable
13
+ source_format: str
14
+ source_version: str
15
+ fixture: Callable | None = None
16
+
17
+
18
+ def csv_rows(path, options):
19
+ rows, _ = read_csv_rows(path, **options)
20
+ return rows
21
+
22
+
23
+ ADAPTERS = Registry("adapters", Adapter, {"csv": Adapter(csv_rows, "CSV", "1")})
caat/audit.py ADDED
@@ -0,0 +1,131 @@
1
+ from math import ceil, isfinite
2
+ from random import Random
3
+ from typing import Iterable
4
+
5
+ from .metrics import grouped_counts
6
+ from .table import GradedRow, validated_rows
7
+
8
+
9
+ def percentile_interval(values):
10
+ if not values:
11
+ return None
12
+ ordered = sorted(values)
13
+ return [ordered[ceil(fraction * len(ordered)) - 1] for fraction in (0.025, 0.975)]
14
+
15
+
16
+ def bootstrap_shifts(pairs, replicates, seed):
17
+ # Resample the full rows so the accepted and whole-set risks remain paired.
18
+ rng = Random(seed)
19
+ risk_deltas, whole_deltas, excess_deltas = [], [], []
20
+ for _ in range(replicates):
21
+ accepted = accepted_delta = total_delta = 0
22
+ for kept, change in rng.choices(pairs, k=len(pairs)):
23
+ total_delta += change
24
+ if kept:
25
+ accepted += 1
26
+ accepted_delta += change
27
+ whole = total_delta / len(pairs)
28
+ whole_deltas.append(whole)
29
+ if accepted:
30
+ risk = accepted_delta / accepted
31
+ risk_deltas.append(risk)
32
+ excess_deltas.append(risk - whole)
33
+ return {
34
+ "risk_change": percentile_interval(risk_deltas),
35
+ "overall_risk_change": percentile_interval(whole_deltas),
36
+ "excess_risk_change": percentile_interval(excess_deltas),
37
+ "replicates_with_acceptance": len(risk_deltas),
38
+ }
39
+
40
+
41
+ def audit(
42
+ rows: Iterable[GradedRow],
43
+ *,
44
+ graders: tuple[str, ...],
45
+ threshold: float,
46
+ risk_target: float | None = None,
47
+ bootstrap: int | None = None,
48
+ seed: int = 42,
49
+ ) -> dict:
50
+ """Replay one cutoff against paired correctness outcomes on the same outputs."""
51
+ if not isfinite(threshold):
52
+ raise ValueError("threshold must be finite")
53
+ if risk_target is not None and not 0 <= risk_target <= 1:
54
+ raise ValueError("risk target must be between 0 and 1")
55
+ if bootstrap is not None and bootstrap < 1:
56
+ raise ValueError("bootstrap count must be positive")
57
+ rows = validated_rows(rows, graders, minimum_graders=2)
58
+
59
+ accepted = [row for row in rows if row.score >= threshold]
60
+ results = {}
61
+ for index, name in enumerate(graders):
62
+ overall, groups = grouped_counts(
63
+ (row.group, row.score >= threshold, not row.outcomes[index]) for row in rows
64
+ )
65
+ risk = overall.error_rate
66
+ results[name] = {
67
+ **overall.metrics(),
68
+ "meets_target": risk <= risk_target
69
+ if risk is not None and risk_target is not None
70
+ else None,
71
+ "by_group": {group: counts.metrics() for group, counts in groups.items()},
72
+ }
73
+
74
+ comparisons = {}
75
+ for index, name in enumerate(graders[1:], start=1):
76
+ newly_failed = sum(
77
+ row.outcomes[0] and not row.outcomes[index] for row in accepted
78
+ )
79
+ newly_passed = sum(
80
+ not row.outcomes[0] and row.outcomes[index] for row in accepted
81
+ )
82
+ primary = results[graders[0]]["meets_target"]
83
+ alternate = results[name]["meets_target"]
84
+ risk_change = (
85
+ (newly_failed - newly_passed) / len(accepted) if accepted else None
86
+ )
87
+ changes = [
88
+ (row.score >= threshold, row.outcomes[0] - row.outcomes[index])
89
+ for row in rows
90
+ ]
91
+ overall_change = sum(delta for _, delta in changes) / len(rows)
92
+ comparisons[name] = {
93
+ "newly_failed": newly_failed,
94
+ "newly_passed": newly_passed,
95
+ "risk_change": risk_change,
96
+ "overall_risk_change": overall_change,
97
+ "excess_risk_change": risk_change - overall_change
98
+ if risk_change is not None
99
+ else None,
100
+ "target_decision_changed": primary != alternate
101
+ if primary is not None
102
+ else None,
103
+ }
104
+ if bootstrap is not None:
105
+ comparisons[name]["bootstrap_95"] = bootstrap_shifts(
106
+ changes, bootstrap, seed
107
+ )
108
+
109
+ discordant = [
110
+ row for row in rows if any(bit != row.outcomes[0] for bit in row.outcomes[1:])
111
+ ]
112
+ report = {
113
+ "schema_version": 1,
114
+ "threshold": threshold,
115
+ "risk_target": risk_target,
116
+ "primary_grader": graders[0],
117
+ "graders": results,
118
+ "accepted_comparisons": comparisons,
119
+ "discordant": [
120
+ {
121
+ "id": row.id,
122
+ "group": row.group,
123
+ "accepted": row.score >= threshold,
124
+ "outcomes": dict(zip(graders, row.outcomes, strict=True)),
125
+ }
126
+ for row in sorted(discordant, key=lambda row: row.id)
127
+ ],
128
+ }
129
+ if bootstrap is not None:
130
+ report["bootstrap"] = {"replicates": bootstrap, "seed": seed}
131
+ return report
caat/calibrate.py ADDED
@@ -0,0 +1,328 @@
1
+ """Fit and replay binary correctness probability maps."""
2
+
3
+ import csv
4
+ import json
5
+ from bisect import bisect_right
6
+ from dataclasses import dataclass
7
+ from itertools import groupby
8
+ from math import exp, fsum, isfinite, log, log1p
9
+ from operator import itemgetter
10
+ from pathlib import Path
11
+ from typing import Callable
12
+
13
+ from .metrics import probability_bin
14
+ from .registry import Registry
15
+ from .table import open_predictions_csv, parse_score
16
+
17
+
18
+ def sigmoid(value):
19
+ tail = exp(-abs(value))
20
+ return 1 / (1 + tail) if value >= 0 else tail / (1 + tail)
21
+
22
+
23
+ def logit(probability):
24
+ probability = min(max(probability, 1e-12), 1 - 1e-12)
25
+ return log(probability) - log1p(-probability)
26
+
27
+
28
+ def input_values(score, scale):
29
+ score = float(score)
30
+ if not isfinite(score):
31
+ raise ValueError("calibration scores must be finite")
32
+ if scale == "probability" and not 0 <= score <= 1:
33
+ raise ValueError("calibration probabilities must be between 0 and 1")
34
+ return score
35
+
36
+
37
+ def fit_temperature(scores, outcomes, *, bins=10):
38
+ # The derivative of the binary log loss increases with inverse temperature.
39
+ if not any(scores):
40
+ return {"inverse_temperature": 1.0}
41
+ lo, hi = 1e-6, 100.0
42
+ scale = max(map(abs, scores))
43
+ for _ in range(60):
44
+ midpoint = (lo + hi) / 2
45
+ gradient = fsum(
46
+ (x / scale) * (sigmoid(x * midpoint) - y)
47
+ for x, y in zip(scores, outcomes, strict=True)
48
+ )
49
+ if gradient < 0:
50
+ lo = midpoint
51
+ else:
52
+ hi = midpoint
53
+ return {"inverse_temperature": (lo + hi) / 2}
54
+
55
+
56
+ def fit_platt(scores, outcomes, *, bins=10):
57
+ scale = max(map(abs, scores)) or 1.0
58
+ pairs = [(score / scale, y) for score, y in zip(scores, outcomes, strict=True)]
59
+ positive = sum(outcomes)
60
+ intercept = log(positive + 1) - log(len(outcomes) - positive + 1)
61
+ if positive in (0, len(outcomes)):
62
+ return {"scale": scale, "slope": 0.0, "intercept": intercept}
63
+
64
+ def loss(slope, intercept):
65
+ logits = [(slope * x + intercept) * (1 - 2 * y) for x, y in pairs]
66
+ return slope**2 / 2 + fsum(max(z, 0) + log1p(exp(-abs(z))) for z in logits)
67
+
68
+ slope = 0.0
69
+ for _ in range(100):
70
+ probabilities = [sigmoid(slope * x + intercept) for x, _ in pairs]
71
+ residuals = [p - y for (_, y), p in zip(pairs, probabilities, strict=True)]
72
+ weights = [p * (1 - p) for p in probabilities]
73
+ gradient_slope = slope + fsum(
74
+ x * r for (x, _), r in zip(pairs, residuals, strict=True)
75
+ )
76
+ gradient_intercept = fsum(residuals)
77
+ if max(abs(gradient_slope), abs(gradient_intercept)) < 1e-9:
78
+ break
79
+ ss = 1 + fsum(x * x * w for (x, _), w in zip(pairs, weights, strict=True))
80
+ si = fsum(x * w for (x, _), w in zip(pairs, weights, strict=True))
81
+ ii = fsum(weights)
82
+ determinant = ss * ii - si**2
83
+ ds = (ii * gradient_slope - si * gradient_intercept) / determinant
84
+ di = (ss * gradient_intercept - si * gradient_slope) / determinant
85
+ previous = loss(slope, intercept)
86
+ step = 1.0
87
+ while step > 1e-8 and loss(slope - step * ds, intercept - step * di) > previous:
88
+ step /= 2
89
+ slope -= step * ds
90
+ intercept -= step * di
91
+ return {"scale": scale, "slope": slope, "intercept": intercept}
92
+
93
+
94
+ def fit_isotonic(scores, outcomes, *, bins=10):
95
+ knots, blocks = [], []
96
+ pairs = sorted(zip(scores, outcomes, strict=True), key=itemgetter(0))
97
+ for score, tied in groupby(pairs, key=itemgetter(0)):
98
+ labels = [y for _, y in tied]
99
+ knots.append(score)
100
+ blocks.append((1, len(labels), sum(labels)))
101
+ while len(blocks) > 1:
102
+ n_left, count_left, positives_left = blocks[-2]
103
+ n_right, count_right, positives_right = blocks[-1]
104
+ if positives_left * count_right <= positives_right * count_left:
105
+ break
106
+ blocks[-2:] = [
107
+ (
108
+ n_left + n_right,
109
+ count_left + count_right,
110
+ positives_left + positives_right,
111
+ )
112
+ ]
113
+ return {
114
+ "scores": knots,
115
+ "probabilities": [
116
+ positives / count for size, count, positives in blocks for _ in range(size)
117
+ ],
118
+ }
119
+
120
+
121
+ def fit_histogram(scores, outcomes, *, bins=10):
122
+ totals, positive = [0] * bins, [0] * bins
123
+ for score, outcome in zip(scores, outcomes, strict=True):
124
+ index = probability_bin(score, bins)
125
+ totals[index] += 1
126
+ positive[index] += outcome
127
+ rate = sum(positive) / len(scores)
128
+ return {
129
+ "probabilities": [
130
+ p / n if n else rate for p, n in zip(positive, totals, strict=True)
131
+ ]
132
+ }
133
+
134
+
135
+ @dataclass(frozen=True)
136
+ class Calibrator:
137
+ fit: Callable
138
+ restore: Callable
139
+ monotone: bool
140
+ input_scale: str = "score"
141
+ logits: bool = False
142
+ probability_only: bool = False
143
+
144
+
145
+ def finite_parameter(value):
146
+ if type(value) not in (float, int) or not isfinite(value):
147
+ raise ValueError("map parameters must be finite numbers")
148
+ return value
149
+
150
+
151
+ def saved_probabilities(params):
152
+ values = [finite_parameter(y) for y in params["probabilities"]]
153
+ if not values or any(not 0 <= y <= 1 for y in values):
154
+ raise ValueError("map needs probabilities between 0 and 1")
155
+ return values
156
+
157
+
158
+ def restore_temperature(params):
159
+ slope = finite_parameter(params["inverse_temperature"])
160
+ if slope <= 0:
161
+ raise ValueError("inverse temperature must be positive")
162
+ return lambda x: sigmoid(x * slope)
163
+
164
+
165
+ def restore_platt(params):
166
+ scale, slope, intercept = (
167
+ finite_parameter(params[key]) for key in ("scale", "slope", "intercept")
168
+ )
169
+ if scale <= 0:
170
+ raise ValueError("Platt scale must be positive")
171
+ return lambda x: sigmoid((x / scale) * slope + intercept if slope else intercept)
172
+
173
+
174
+ def restore_histogram(params):
175
+ values = saved_probabilities(params)
176
+ return lambda x: values[probability_bin(x, len(values))]
177
+
178
+
179
+ def restore_isotonic(params):
180
+ values = saved_probabilities(params)
181
+ knots = [finite_parameter(x) for x in params["scores"]]
182
+ if (
183
+ len(knots) != len(values)
184
+ or any(a >= b for a, b in zip(knots, knots[1:]))
185
+ or values != sorted(values)
186
+ ):
187
+ raise ValueError("invalid isotonic knots")
188
+
189
+ def interpolate(x):
190
+ right = bisect_right(knots, x)
191
+ if right == 0:
192
+ return values[0]
193
+ if right == len(knots):
194
+ return values[-1]
195
+ left = right - 1
196
+ a, b = knots[left], knots[right]
197
+ fraction = (
198
+ (x / 2 - a / 2) / (b / 2 - a / 2)
199
+ if b - a == float("inf")
200
+ else (x - a) / (b - a)
201
+ )
202
+ return values[left] + fraction * (values[right] - values[left])
203
+
204
+ return interpolate
205
+
206
+
207
+ CALIBRATORS = Registry(
208
+ "calibrators",
209
+ Calibrator,
210
+ {
211
+ "temperature": Calibrator(
212
+ fit_temperature, restore_temperature, True, "probability", logits=True
213
+ ),
214
+ "platt": Calibrator(fit_platt, restore_platt, False, logits=True),
215
+ "isotonic": Calibrator(fit_isotonic, restore_isotonic, True),
216
+ "histogram": Calibrator(
217
+ fit_histogram,
218
+ restore_histogram,
219
+ False,
220
+ "probability",
221
+ probability_only=True,
222
+ ),
223
+ },
224
+ )
225
+
226
+
227
+ def default_scale(method):
228
+ if method not in CALIBRATORS:
229
+ raise ValueError(f"unknown calibration method: {method!r}")
230
+ return CALIBRATORS[method].input_scale
231
+
232
+
233
+ def check_scale(component, input_scale):
234
+ if input_scale not in {"score", "probability"}:
235
+ raise ValueError("input scale must be score or probability")
236
+ if component.probability_only and input_scale != "probability":
237
+ raise ValueError("this calibration method requires probabilities")
238
+
239
+
240
+ def fit_calibration(scores, outcomes, *, method, input_scale=None, bins=10):
241
+ """Return a JSON-serializable map trained only on the supplied rows."""
242
+ if method not in CALIBRATORS:
243
+ raise ValueError(f"unknown calibration method: {method!r}")
244
+ component = CALIBRATORS[method]
245
+ input_scale = input_scale or component.input_scale
246
+ check_scale(component, input_scale)
247
+ if type(bins) is not int or bins < 1:
248
+ raise ValueError("bins must be a positive integer")
249
+ pairs = sorted(zip(scores, outcomes, strict=True))
250
+ if not pairs or any(y not in (0, 1) for _, y in pairs):
251
+ raise ValueError("calibration requires rows with binary correctness outcomes")
252
+ scores = [input_values(x, input_scale) for x, _ in pairs]
253
+ outcomes = [int(y) for _, y in pairs]
254
+ if input_scale == "probability" and component.logits:
255
+ scores = [logit(x) for x in scores]
256
+ params = component.fit(scores, outcomes, bins=bins)
257
+ if not isinstance(params, dict):
258
+ raise ValueError("a calibrator must return a dictionary of map parameters")
259
+ return {
260
+ "schema_version": 1,
261
+ "method": method,
262
+ "input_scale": input_scale,
263
+ "parameters": params,
264
+ }
265
+
266
+
267
+ def calibration_transform(mapping):
268
+ """Validate a saved map and return its score-to-probability function."""
269
+
270
+ try:
271
+ if mapping["schema_version"] != 1:
272
+ raise ValueError("unsupported calibration map schema")
273
+ method, scale, params = (
274
+ mapping[k] for k in ("method", "input_scale", "parameters")
275
+ )
276
+ if method not in CALIBRATORS:
277
+ raise ValueError(f"unknown calibration method: {method!r}")
278
+ component = CALIBRATORS[method]
279
+ check_scale(component, scale)
280
+ predict = component.restore(params)
281
+ if not callable(predict):
282
+ raise ValueError(
283
+ "a calibrator must restore a score-to-probability callable"
284
+ )
285
+ except (KeyError, TypeError, OverflowError) as exc:
286
+ raise ValueError("invalid calibration map") from exc
287
+
288
+ def transform(score):
289
+ x = input_values(score, scale)
290
+ if component.logits and scale == "probability":
291
+ x = logit(x)
292
+ probability = float(predict(x))
293
+ if not 0 <= probability <= 1:
294
+ raise ValueError("a calibrated probability must be between 0 and 1")
295
+ return probability
296
+
297
+ return transform
298
+
299
+
300
+ def apply_csv(path, output, mapping, score_column, probability_column):
301
+ transform = calibration_transform(mapping)
302
+ with open_predictions_csv(path, {score_column}, None) as (reader, _):
303
+ fields = reader.fieldnames
304
+ if probability_column in fields or not probability_column:
305
+ raise ValueError("probability output column must be new and nonempty")
306
+ rows = []
307
+ for row in reader:
308
+ if None in row or None in row.values():
309
+ raise ValueError(
310
+ f"incomplete or extra CSV fields at line {reader.line_num}"
311
+ )
312
+ row[probability_column] = transform(
313
+ parse_score(row[score_column], reader.line_num)
314
+ )
315
+ rows.append(row)
316
+ if not rows:
317
+ raise ValueError("at least one prediction is required")
318
+ output = Path(output)
319
+ output.parent.mkdir(parents=True, exist_ok=True)
320
+ with output.open("w", newline="", encoding="utf-8") as destination:
321
+ writer = csv.DictWriter(destination, fieldnames=[*fields, probability_column])
322
+ writer.writeheader()
323
+ writer.writerows(rows)
324
+ return len(rows)
325
+
326
+
327
+ def read_calibration(path):
328
+ return json.loads(path.read_text(encoding="utf-8"))