caat-annotation 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- caat/__init__.py +9 -0
- caat/adapters.py +23 -0
- caat/audit.py +131 -0
- caat/calibrate.py +328 -0
- caat/cli.py +761 -0
- caat/collect.py +205 -0
- caat/evaluate.py +66 -0
- caat/features.py +304 -0
- caat/html_report.py +130 -0
- caat/metrics.py +196 -0
- caat/plots.py +223 -0
- caat/plugins.py +32 -0
- caat/policy.py +127 -0
- caat/queue.py +75 -0
- caat/registry.py +71 -0
- caat/renderers.py +64 -0
- caat/report.py +209 -0
- caat/rules.py +127 -0
- caat/scores.py +231 -0
- caat/select.py +88 -0
- caat/table.py +236 -0
- caat_annotation-0.2.0.dist-info/METADATA +221 -0
- caat_annotation-0.2.0.dist-info/RECORD +27 -0
- caat_annotation-0.2.0.dist-info/WHEEL +5 -0
- caat_annotation-0.2.0.dist-info/entry_points.txt +2 -0
- caat_annotation-0.2.0.dist-info/licenses/LICENSE +21 -0
- caat_annotation-0.2.0.dist-info/top_level.txt +1 -0
caat/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""Confidence-Aware Annotation Toolkit."""
|
|
2
|
+
|
|
3
|
+
from .audit import audit
|
|
4
|
+
from .evaluate import evaluate
|
|
5
|
+
from .policy import policy
|
|
6
|
+
from .select import select
|
|
7
|
+
from .table import GradedRow, load_csv
|
|
8
|
+
|
|
9
|
+
__all__ = ["GradedRow", "audit", "evaluate", "load_csv", "policy", "select"]
|
caat/adapters.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Input adapters produce graded rows for CAAT's analysis commands."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Callable
|
|
5
|
+
|
|
6
|
+
from .registry import Registry
|
|
7
|
+
from .table import read_csv_rows
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class Adapter:
|
|
12
|
+
read: Callable
|
|
13
|
+
source_format: str
|
|
14
|
+
source_version: str
|
|
15
|
+
fixture: Callable | None = None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def csv_rows(path, options):
|
|
19
|
+
rows, _ = read_csv_rows(path, **options)
|
|
20
|
+
return rows
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
ADAPTERS = Registry("adapters", Adapter, {"csv": Adapter(csv_rows, "CSV", "1")})
|
caat/audit.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
from math import ceil, isfinite
|
|
2
|
+
from random import Random
|
|
3
|
+
from typing import Iterable
|
|
4
|
+
|
|
5
|
+
from .metrics import grouped_counts
|
|
6
|
+
from .table import GradedRow, validated_rows
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def percentile_interval(values):
|
|
10
|
+
if not values:
|
|
11
|
+
return None
|
|
12
|
+
ordered = sorted(values)
|
|
13
|
+
return [ordered[ceil(fraction * len(ordered)) - 1] for fraction in (0.025, 0.975)]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def bootstrap_shifts(pairs, replicates, seed):
|
|
17
|
+
# Resample the full rows so the accepted and whole-set risks remain paired.
|
|
18
|
+
rng = Random(seed)
|
|
19
|
+
risk_deltas, whole_deltas, excess_deltas = [], [], []
|
|
20
|
+
for _ in range(replicates):
|
|
21
|
+
accepted = accepted_delta = total_delta = 0
|
|
22
|
+
for kept, change in rng.choices(pairs, k=len(pairs)):
|
|
23
|
+
total_delta += change
|
|
24
|
+
if kept:
|
|
25
|
+
accepted += 1
|
|
26
|
+
accepted_delta += change
|
|
27
|
+
whole = total_delta / len(pairs)
|
|
28
|
+
whole_deltas.append(whole)
|
|
29
|
+
if accepted:
|
|
30
|
+
risk = accepted_delta / accepted
|
|
31
|
+
risk_deltas.append(risk)
|
|
32
|
+
excess_deltas.append(risk - whole)
|
|
33
|
+
return {
|
|
34
|
+
"risk_change": percentile_interval(risk_deltas),
|
|
35
|
+
"overall_risk_change": percentile_interval(whole_deltas),
|
|
36
|
+
"excess_risk_change": percentile_interval(excess_deltas),
|
|
37
|
+
"replicates_with_acceptance": len(risk_deltas),
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def audit(
|
|
42
|
+
rows: Iterable[GradedRow],
|
|
43
|
+
*,
|
|
44
|
+
graders: tuple[str, ...],
|
|
45
|
+
threshold: float,
|
|
46
|
+
risk_target: float | None = None,
|
|
47
|
+
bootstrap: int | None = None,
|
|
48
|
+
seed: int = 42,
|
|
49
|
+
) -> dict:
|
|
50
|
+
"""Replay one cutoff against paired correctness outcomes on the same outputs."""
|
|
51
|
+
if not isfinite(threshold):
|
|
52
|
+
raise ValueError("threshold must be finite")
|
|
53
|
+
if risk_target is not None and not 0 <= risk_target <= 1:
|
|
54
|
+
raise ValueError("risk target must be between 0 and 1")
|
|
55
|
+
if bootstrap is not None and bootstrap < 1:
|
|
56
|
+
raise ValueError("bootstrap count must be positive")
|
|
57
|
+
rows = validated_rows(rows, graders, minimum_graders=2)
|
|
58
|
+
|
|
59
|
+
accepted = [row for row in rows if row.score >= threshold]
|
|
60
|
+
results = {}
|
|
61
|
+
for index, name in enumerate(graders):
|
|
62
|
+
overall, groups = grouped_counts(
|
|
63
|
+
(row.group, row.score >= threshold, not row.outcomes[index]) for row in rows
|
|
64
|
+
)
|
|
65
|
+
risk = overall.error_rate
|
|
66
|
+
results[name] = {
|
|
67
|
+
**overall.metrics(),
|
|
68
|
+
"meets_target": risk <= risk_target
|
|
69
|
+
if risk is not None and risk_target is not None
|
|
70
|
+
else None,
|
|
71
|
+
"by_group": {group: counts.metrics() for group, counts in groups.items()},
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
comparisons = {}
|
|
75
|
+
for index, name in enumerate(graders[1:], start=1):
|
|
76
|
+
newly_failed = sum(
|
|
77
|
+
row.outcomes[0] and not row.outcomes[index] for row in accepted
|
|
78
|
+
)
|
|
79
|
+
newly_passed = sum(
|
|
80
|
+
not row.outcomes[0] and row.outcomes[index] for row in accepted
|
|
81
|
+
)
|
|
82
|
+
primary = results[graders[0]]["meets_target"]
|
|
83
|
+
alternate = results[name]["meets_target"]
|
|
84
|
+
risk_change = (
|
|
85
|
+
(newly_failed - newly_passed) / len(accepted) if accepted else None
|
|
86
|
+
)
|
|
87
|
+
changes = [
|
|
88
|
+
(row.score >= threshold, row.outcomes[0] - row.outcomes[index])
|
|
89
|
+
for row in rows
|
|
90
|
+
]
|
|
91
|
+
overall_change = sum(delta for _, delta in changes) / len(rows)
|
|
92
|
+
comparisons[name] = {
|
|
93
|
+
"newly_failed": newly_failed,
|
|
94
|
+
"newly_passed": newly_passed,
|
|
95
|
+
"risk_change": risk_change,
|
|
96
|
+
"overall_risk_change": overall_change,
|
|
97
|
+
"excess_risk_change": risk_change - overall_change
|
|
98
|
+
if risk_change is not None
|
|
99
|
+
else None,
|
|
100
|
+
"target_decision_changed": primary != alternate
|
|
101
|
+
if primary is not None
|
|
102
|
+
else None,
|
|
103
|
+
}
|
|
104
|
+
if bootstrap is not None:
|
|
105
|
+
comparisons[name]["bootstrap_95"] = bootstrap_shifts(
|
|
106
|
+
changes, bootstrap, seed
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
discordant = [
|
|
110
|
+
row for row in rows if any(bit != row.outcomes[0] for bit in row.outcomes[1:])
|
|
111
|
+
]
|
|
112
|
+
report = {
|
|
113
|
+
"schema_version": 1,
|
|
114
|
+
"threshold": threshold,
|
|
115
|
+
"risk_target": risk_target,
|
|
116
|
+
"primary_grader": graders[0],
|
|
117
|
+
"graders": results,
|
|
118
|
+
"accepted_comparisons": comparisons,
|
|
119
|
+
"discordant": [
|
|
120
|
+
{
|
|
121
|
+
"id": row.id,
|
|
122
|
+
"group": row.group,
|
|
123
|
+
"accepted": row.score >= threshold,
|
|
124
|
+
"outcomes": dict(zip(graders, row.outcomes, strict=True)),
|
|
125
|
+
}
|
|
126
|
+
for row in sorted(discordant, key=lambda row: row.id)
|
|
127
|
+
],
|
|
128
|
+
}
|
|
129
|
+
if bootstrap is not None:
|
|
130
|
+
report["bootstrap"] = {"replicates": bootstrap, "seed": seed}
|
|
131
|
+
return report
|
caat/calibrate.py
ADDED
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
"""Fit and replay binary correctness probability maps."""
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
import json
|
|
5
|
+
from bisect import bisect_right
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from itertools import groupby
|
|
8
|
+
from math import exp, fsum, isfinite, log, log1p
|
|
9
|
+
from operator import itemgetter
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Callable
|
|
12
|
+
|
|
13
|
+
from .metrics import probability_bin
|
|
14
|
+
from .registry import Registry
|
|
15
|
+
from .table import open_predictions_csv, parse_score
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def sigmoid(value):
|
|
19
|
+
tail = exp(-abs(value))
|
|
20
|
+
return 1 / (1 + tail) if value >= 0 else tail / (1 + tail)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def logit(probability):
|
|
24
|
+
probability = min(max(probability, 1e-12), 1 - 1e-12)
|
|
25
|
+
return log(probability) - log1p(-probability)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def input_values(score, scale):
|
|
29
|
+
score = float(score)
|
|
30
|
+
if not isfinite(score):
|
|
31
|
+
raise ValueError("calibration scores must be finite")
|
|
32
|
+
if scale == "probability" and not 0 <= score <= 1:
|
|
33
|
+
raise ValueError("calibration probabilities must be between 0 and 1")
|
|
34
|
+
return score
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def fit_temperature(scores, outcomes, *, bins=10):
|
|
38
|
+
# The derivative of the binary log loss increases with inverse temperature.
|
|
39
|
+
if not any(scores):
|
|
40
|
+
return {"inverse_temperature": 1.0}
|
|
41
|
+
lo, hi = 1e-6, 100.0
|
|
42
|
+
scale = max(map(abs, scores))
|
|
43
|
+
for _ in range(60):
|
|
44
|
+
midpoint = (lo + hi) / 2
|
|
45
|
+
gradient = fsum(
|
|
46
|
+
(x / scale) * (sigmoid(x * midpoint) - y)
|
|
47
|
+
for x, y in zip(scores, outcomes, strict=True)
|
|
48
|
+
)
|
|
49
|
+
if gradient < 0:
|
|
50
|
+
lo = midpoint
|
|
51
|
+
else:
|
|
52
|
+
hi = midpoint
|
|
53
|
+
return {"inverse_temperature": (lo + hi) / 2}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def fit_platt(scores, outcomes, *, bins=10):
|
|
57
|
+
scale = max(map(abs, scores)) or 1.0
|
|
58
|
+
pairs = [(score / scale, y) for score, y in zip(scores, outcomes, strict=True)]
|
|
59
|
+
positive = sum(outcomes)
|
|
60
|
+
intercept = log(positive + 1) - log(len(outcomes) - positive + 1)
|
|
61
|
+
if positive in (0, len(outcomes)):
|
|
62
|
+
return {"scale": scale, "slope": 0.0, "intercept": intercept}
|
|
63
|
+
|
|
64
|
+
def loss(slope, intercept):
|
|
65
|
+
logits = [(slope * x + intercept) * (1 - 2 * y) for x, y in pairs]
|
|
66
|
+
return slope**2 / 2 + fsum(max(z, 0) + log1p(exp(-abs(z))) for z in logits)
|
|
67
|
+
|
|
68
|
+
slope = 0.0
|
|
69
|
+
for _ in range(100):
|
|
70
|
+
probabilities = [sigmoid(slope * x + intercept) for x, _ in pairs]
|
|
71
|
+
residuals = [p - y for (_, y), p in zip(pairs, probabilities, strict=True)]
|
|
72
|
+
weights = [p * (1 - p) for p in probabilities]
|
|
73
|
+
gradient_slope = slope + fsum(
|
|
74
|
+
x * r for (x, _), r in zip(pairs, residuals, strict=True)
|
|
75
|
+
)
|
|
76
|
+
gradient_intercept = fsum(residuals)
|
|
77
|
+
if max(abs(gradient_slope), abs(gradient_intercept)) < 1e-9:
|
|
78
|
+
break
|
|
79
|
+
ss = 1 + fsum(x * x * w for (x, _), w in zip(pairs, weights, strict=True))
|
|
80
|
+
si = fsum(x * w for (x, _), w in zip(pairs, weights, strict=True))
|
|
81
|
+
ii = fsum(weights)
|
|
82
|
+
determinant = ss * ii - si**2
|
|
83
|
+
ds = (ii * gradient_slope - si * gradient_intercept) / determinant
|
|
84
|
+
di = (ss * gradient_intercept - si * gradient_slope) / determinant
|
|
85
|
+
previous = loss(slope, intercept)
|
|
86
|
+
step = 1.0
|
|
87
|
+
while step > 1e-8 and loss(slope - step * ds, intercept - step * di) > previous:
|
|
88
|
+
step /= 2
|
|
89
|
+
slope -= step * ds
|
|
90
|
+
intercept -= step * di
|
|
91
|
+
return {"scale": scale, "slope": slope, "intercept": intercept}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def fit_isotonic(scores, outcomes, *, bins=10):
|
|
95
|
+
knots, blocks = [], []
|
|
96
|
+
pairs = sorted(zip(scores, outcomes, strict=True), key=itemgetter(0))
|
|
97
|
+
for score, tied in groupby(pairs, key=itemgetter(0)):
|
|
98
|
+
labels = [y for _, y in tied]
|
|
99
|
+
knots.append(score)
|
|
100
|
+
blocks.append((1, len(labels), sum(labels)))
|
|
101
|
+
while len(blocks) > 1:
|
|
102
|
+
n_left, count_left, positives_left = blocks[-2]
|
|
103
|
+
n_right, count_right, positives_right = blocks[-1]
|
|
104
|
+
if positives_left * count_right <= positives_right * count_left:
|
|
105
|
+
break
|
|
106
|
+
blocks[-2:] = [
|
|
107
|
+
(
|
|
108
|
+
n_left + n_right,
|
|
109
|
+
count_left + count_right,
|
|
110
|
+
positives_left + positives_right,
|
|
111
|
+
)
|
|
112
|
+
]
|
|
113
|
+
return {
|
|
114
|
+
"scores": knots,
|
|
115
|
+
"probabilities": [
|
|
116
|
+
positives / count for size, count, positives in blocks for _ in range(size)
|
|
117
|
+
],
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def fit_histogram(scores, outcomes, *, bins=10):
|
|
122
|
+
totals, positive = [0] * bins, [0] * bins
|
|
123
|
+
for score, outcome in zip(scores, outcomes, strict=True):
|
|
124
|
+
index = probability_bin(score, bins)
|
|
125
|
+
totals[index] += 1
|
|
126
|
+
positive[index] += outcome
|
|
127
|
+
rate = sum(positive) / len(scores)
|
|
128
|
+
return {
|
|
129
|
+
"probabilities": [
|
|
130
|
+
p / n if n else rate for p, n in zip(positive, totals, strict=True)
|
|
131
|
+
]
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@dataclass(frozen=True)
|
|
136
|
+
class Calibrator:
|
|
137
|
+
fit: Callable
|
|
138
|
+
restore: Callable
|
|
139
|
+
monotone: bool
|
|
140
|
+
input_scale: str = "score"
|
|
141
|
+
logits: bool = False
|
|
142
|
+
probability_only: bool = False
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def finite_parameter(value):
|
|
146
|
+
if type(value) not in (float, int) or not isfinite(value):
|
|
147
|
+
raise ValueError("map parameters must be finite numbers")
|
|
148
|
+
return value
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def saved_probabilities(params):
|
|
152
|
+
values = [finite_parameter(y) for y in params["probabilities"]]
|
|
153
|
+
if not values or any(not 0 <= y <= 1 for y in values):
|
|
154
|
+
raise ValueError("map needs probabilities between 0 and 1")
|
|
155
|
+
return values
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def restore_temperature(params):
|
|
159
|
+
slope = finite_parameter(params["inverse_temperature"])
|
|
160
|
+
if slope <= 0:
|
|
161
|
+
raise ValueError("inverse temperature must be positive")
|
|
162
|
+
return lambda x: sigmoid(x * slope)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def restore_platt(params):
|
|
166
|
+
scale, slope, intercept = (
|
|
167
|
+
finite_parameter(params[key]) for key in ("scale", "slope", "intercept")
|
|
168
|
+
)
|
|
169
|
+
if scale <= 0:
|
|
170
|
+
raise ValueError("Platt scale must be positive")
|
|
171
|
+
return lambda x: sigmoid((x / scale) * slope + intercept if slope else intercept)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def restore_histogram(params):
|
|
175
|
+
values = saved_probabilities(params)
|
|
176
|
+
return lambda x: values[probability_bin(x, len(values))]
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def restore_isotonic(params):
|
|
180
|
+
values = saved_probabilities(params)
|
|
181
|
+
knots = [finite_parameter(x) for x in params["scores"]]
|
|
182
|
+
if (
|
|
183
|
+
len(knots) != len(values)
|
|
184
|
+
or any(a >= b for a, b in zip(knots, knots[1:]))
|
|
185
|
+
or values != sorted(values)
|
|
186
|
+
):
|
|
187
|
+
raise ValueError("invalid isotonic knots")
|
|
188
|
+
|
|
189
|
+
def interpolate(x):
|
|
190
|
+
right = bisect_right(knots, x)
|
|
191
|
+
if right == 0:
|
|
192
|
+
return values[0]
|
|
193
|
+
if right == len(knots):
|
|
194
|
+
return values[-1]
|
|
195
|
+
left = right - 1
|
|
196
|
+
a, b = knots[left], knots[right]
|
|
197
|
+
fraction = (
|
|
198
|
+
(x / 2 - a / 2) / (b / 2 - a / 2)
|
|
199
|
+
if b - a == float("inf")
|
|
200
|
+
else (x - a) / (b - a)
|
|
201
|
+
)
|
|
202
|
+
return values[left] + fraction * (values[right] - values[left])
|
|
203
|
+
|
|
204
|
+
return interpolate
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
CALIBRATORS = Registry(
|
|
208
|
+
"calibrators",
|
|
209
|
+
Calibrator,
|
|
210
|
+
{
|
|
211
|
+
"temperature": Calibrator(
|
|
212
|
+
fit_temperature, restore_temperature, True, "probability", logits=True
|
|
213
|
+
),
|
|
214
|
+
"platt": Calibrator(fit_platt, restore_platt, False, logits=True),
|
|
215
|
+
"isotonic": Calibrator(fit_isotonic, restore_isotonic, True),
|
|
216
|
+
"histogram": Calibrator(
|
|
217
|
+
fit_histogram,
|
|
218
|
+
restore_histogram,
|
|
219
|
+
False,
|
|
220
|
+
"probability",
|
|
221
|
+
probability_only=True,
|
|
222
|
+
),
|
|
223
|
+
},
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def default_scale(method):
|
|
228
|
+
if method not in CALIBRATORS:
|
|
229
|
+
raise ValueError(f"unknown calibration method: {method!r}")
|
|
230
|
+
return CALIBRATORS[method].input_scale
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def check_scale(component, input_scale):
|
|
234
|
+
if input_scale not in {"score", "probability"}:
|
|
235
|
+
raise ValueError("input scale must be score or probability")
|
|
236
|
+
if component.probability_only and input_scale != "probability":
|
|
237
|
+
raise ValueError("this calibration method requires probabilities")
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def fit_calibration(scores, outcomes, *, method, input_scale=None, bins=10):
|
|
241
|
+
"""Return a JSON-serializable map trained only on the supplied rows."""
|
|
242
|
+
if method not in CALIBRATORS:
|
|
243
|
+
raise ValueError(f"unknown calibration method: {method!r}")
|
|
244
|
+
component = CALIBRATORS[method]
|
|
245
|
+
input_scale = input_scale or component.input_scale
|
|
246
|
+
check_scale(component, input_scale)
|
|
247
|
+
if type(bins) is not int or bins < 1:
|
|
248
|
+
raise ValueError("bins must be a positive integer")
|
|
249
|
+
pairs = sorted(zip(scores, outcomes, strict=True))
|
|
250
|
+
if not pairs or any(y not in (0, 1) for _, y in pairs):
|
|
251
|
+
raise ValueError("calibration requires rows with binary correctness outcomes")
|
|
252
|
+
scores = [input_values(x, input_scale) for x, _ in pairs]
|
|
253
|
+
outcomes = [int(y) for _, y in pairs]
|
|
254
|
+
if input_scale == "probability" and component.logits:
|
|
255
|
+
scores = [logit(x) for x in scores]
|
|
256
|
+
params = component.fit(scores, outcomes, bins=bins)
|
|
257
|
+
if not isinstance(params, dict):
|
|
258
|
+
raise ValueError("a calibrator must return a dictionary of map parameters")
|
|
259
|
+
return {
|
|
260
|
+
"schema_version": 1,
|
|
261
|
+
"method": method,
|
|
262
|
+
"input_scale": input_scale,
|
|
263
|
+
"parameters": params,
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def calibration_transform(mapping):
|
|
268
|
+
"""Validate a saved map and return its score-to-probability function."""
|
|
269
|
+
|
|
270
|
+
try:
|
|
271
|
+
if mapping["schema_version"] != 1:
|
|
272
|
+
raise ValueError("unsupported calibration map schema")
|
|
273
|
+
method, scale, params = (
|
|
274
|
+
mapping[k] for k in ("method", "input_scale", "parameters")
|
|
275
|
+
)
|
|
276
|
+
if method not in CALIBRATORS:
|
|
277
|
+
raise ValueError(f"unknown calibration method: {method!r}")
|
|
278
|
+
component = CALIBRATORS[method]
|
|
279
|
+
check_scale(component, scale)
|
|
280
|
+
predict = component.restore(params)
|
|
281
|
+
if not callable(predict):
|
|
282
|
+
raise ValueError(
|
|
283
|
+
"a calibrator must restore a score-to-probability callable"
|
|
284
|
+
)
|
|
285
|
+
except (KeyError, TypeError, OverflowError) as exc:
|
|
286
|
+
raise ValueError("invalid calibration map") from exc
|
|
287
|
+
|
|
288
|
+
def transform(score):
|
|
289
|
+
x = input_values(score, scale)
|
|
290
|
+
if component.logits and scale == "probability":
|
|
291
|
+
x = logit(x)
|
|
292
|
+
probability = float(predict(x))
|
|
293
|
+
if not 0 <= probability <= 1:
|
|
294
|
+
raise ValueError("a calibrated probability must be between 0 and 1")
|
|
295
|
+
return probability
|
|
296
|
+
|
|
297
|
+
return transform
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def apply_csv(path, output, mapping, score_column, probability_column):
|
|
301
|
+
transform = calibration_transform(mapping)
|
|
302
|
+
with open_predictions_csv(path, {score_column}, None) as (reader, _):
|
|
303
|
+
fields = reader.fieldnames
|
|
304
|
+
if probability_column in fields or not probability_column:
|
|
305
|
+
raise ValueError("probability output column must be new and nonempty")
|
|
306
|
+
rows = []
|
|
307
|
+
for row in reader:
|
|
308
|
+
if None in row or None in row.values():
|
|
309
|
+
raise ValueError(
|
|
310
|
+
f"incomplete or extra CSV fields at line {reader.line_num}"
|
|
311
|
+
)
|
|
312
|
+
row[probability_column] = transform(
|
|
313
|
+
parse_score(row[score_column], reader.line_num)
|
|
314
|
+
)
|
|
315
|
+
rows.append(row)
|
|
316
|
+
if not rows:
|
|
317
|
+
raise ValueError("at least one prediction is required")
|
|
318
|
+
output = Path(output)
|
|
319
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
320
|
+
with output.open("w", newline="", encoding="utf-8") as destination:
|
|
321
|
+
writer = csv.DictWriter(destination, fieldnames=[*fields, probability_column])
|
|
322
|
+
writer.writeheader()
|
|
323
|
+
writer.writerows(rows)
|
|
324
|
+
return len(rows)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def read_calibration(path):
|
|
328
|
+
return json.loads(path.read_text(encoding="utf-8"))
|