agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,661 @@
|
|
|
1
|
+
"""`aut label eval`, `aut label choose` and `aut label train-encoder`: the bake-off on gold.
|
|
2
|
+
|
|
3
|
+
Raw scores are saved per task, model, revision and prompt, so a rerun calls the model only for new
|
|
4
|
+
gold units and tuning needs no model. Calibration and the per-label tau fit on four of five folds
|
|
5
|
+
(evalset.fold: same for every model, grouped by day) and report on the fifth; the saved calibration
|
|
6
|
+
and tau then fit on all five. A floor on precision and on recall per label is never relaxed: a
|
|
7
|
+
model that misses one is marked as failing. A stable part of the days (evalset.final_test) stays
|
|
8
|
+
out of all of this; only `aut label eval --final --model M` scores it, into <task>.final.jsonl.
|
|
9
|
+
|
|
10
|
+
Per backend, one eval record of numbers (and model ids) is appended to <task>.eval.jsonl and sent
|
|
11
|
+
as an activity_event of kind `eval`. `aut label eval` prints one table per task (quality, speed,
|
|
12
|
+
memory, with Pareto and failure marks); the owner picks with `aut label choose --task T --model M`,
|
|
13
|
+
which writes that model's saved result into label.json, which the nightly run reads.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
import hashlib
|
|
20
|
+
import math
|
|
21
|
+
import random
|
|
22
|
+
import sys
|
|
23
|
+
import time
|
|
24
|
+
from collections import Counter
|
|
25
|
+
from dataclasses import asdict
|
|
26
|
+
from datetime import date, datetime, timezone
|
|
27
|
+
from typing import Any, Iterable, NamedTuple, Sequence
|
|
28
|
+
|
|
29
|
+
from ..config import LABEL_DAYS, AutError, Config
|
|
30
|
+
from ..event import ActivityEvent, make_event_id
|
|
31
|
+
from ..push import send_activity_rows
|
|
32
|
+
from ..state import atomic_write_text
|
|
33
|
+
from .backends import (MODELS, BackendUnavailable, Encoder, encoder_path, features, find_model,
|
|
34
|
+
make_backend, short_name)
|
|
35
|
+
from .core import (Backend, Choice, Request, calibrate, choice_path, fit_platt, fit_tau,
|
|
36
|
+
fit_temperature, read_choices, softmax, top_label)
|
|
37
|
+
from .evalset import (FLOORS, FOLDS, append_jsonl, eval_path, final_test, fold, gold_latest,
|
|
38
|
+
in_encoder_train_split, read_jsonl, scores_path, silver_by_unit, stratum,
|
|
39
|
+
window_units)
|
|
40
|
+
from .tasks import TASKS, Source
|
|
41
|
+
|
|
42
|
+
SHARE_ERROR_PP = 5.0 # a label's share on known items may be off this much (CONTRACT.md)
|
|
43
|
+
GOLD_WEIGHT = 3.0 # train-encoder: one gold answer counts as three silver ones
|
|
44
|
+
Dist = dict[str, float]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class _Item(NamedTuple):
|
|
48
|
+
"""One gold unit of a bake-off."""
|
|
49
|
+
req: Request
|
|
50
|
+
gold: str
|
|
51
|
+
weight: float
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _round4(x: float | None) -> float | None:
|
|
55
|
+
return None if x is None else round(x, 4)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def wilson_lb(k: int, n: int, z: float = 1.96) -> float | None:
|
|
59
|
+
"""Lower bound of the 95% Wilson interval of k right out of n."""
|
|
60
|
+
if not n:
|
|
61
|
+
return None
|
|
62
|
+
p = k / n
|
|
63
|
+
return (p + z * z / (2 * n) - z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n))) \
|
|
64
|
+
/ (1 + z * z / n)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def kappa(pairs: list[tuple[str, str]]) -> float | None:
|
|
68
|
+
"""Cohen's kappa of (predicted, gold)."""
|
|
69
|
+
if not pairs:
|
|
70
|
+
return None
|
|
71
|
+
n = len(pairs)
|
|
72
|
+
po = sum(a == b for a, b in pairs) / n
|
|
73
|
+
pa, pb = Counter(a for a, _ in pairs), Counter(b for _, b in pairs)
|
|
74
|
+
pe = sum(pa[x] * pb[x] for x in pa) / (n * n)
|
|
75
|
+
return 1.0 if pe == 1 else (po - pe) / (1 - pe)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def ece(tops: list[tuple[float, bool]], bins: int = 10) -> float | None:
|
|
79
|
+
"""Expected calibration error of (top probability, was right), equal-width bins."""
|
|
80
|
+
if not tops:
|
|
81
|
+
return None
|
|
82
|
+
by: dict[int, list[Any]] = {}
|
|
83
|
+
for p, hit in tops:
|
|
84
|
+
by.setdefault(min(int(p * bins), bins - 1), []).append((p, hit))
|
|
85
|
+
return sum(abs(sum(p for p, _ in b) / len(b) - sum(h for _, h in b) / len(b)) * len(b)
|
|
86
|
+
for b in by.values()) / len(tops)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def weights(task: str, golds: list[str]) -> dict[str, float]:
|
|
90
|
+
"""unit_id -> weight that re-weights the stratified gold sample to natural frequency: the
|
|
91
|
+
share of its silver stratum among every silver unit over its share among the gold units.
|
|
92
|
+
shortcut: the silver sample stands in for the population (teach samples by day and unit
|
|
93
|
+
type, not by label); upgrade by counting nightly rows once a task runs nightly."""
|
|
94
|
+
silver = silver_by_unit(task)
|
|
95
|
+
pop = Counter(stratum(t) for t in silver.values())
|
|
96
|
+
sample = Counter(stratum(silver.get(u)) for u in golds)
|
|
97
|
+
tp, ts = sum(pop.values()), len(golds)
|
|
98
|
+
return {u: (pop[s] / tp) / (sample[s] / ts) if pop[s] else 1.0
|
|
99
|
+
for u in golds for s in [stratum(silver.get(u))]}
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def brier(rows: list[tuple[Dist, str]]) -> float:
|
|
103
|
+
"""Mean over (distribution, gold) of the squared error summed over candidates: 0 to 2."""
|
|
104
|
+
return sum(sum((p - (k == g)) ** 2 for k, p in d.items()) for d, g in rows) / len(rows)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def nll(rows: list[tuple[Dist, str]]) -> float:
|
|
108
|
+
return -sum(math.log(max(d[g], 1e-12)) for d, g in rows) / len(rows)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def reliability(tops: list[tuple[float, bool]], size: int = 20) -> list[dict[str, Any]]:
|
|
112
|
+
"""(top probability, was right), lowest first, in groups of about `size`: mean p, share
|
|
113
|
+
right."""
|
|
114
|
+
tops = sorted(tops)
|
|
115
|
+
groups = [tops[i:i + size] for i in range(0, len(tops), size)]
|
|
116
|
+
if len(groups) > 1 and len(groups[-1]) < size / 2:
|
|
117
|
+
groups[-2:] = [groups[-2] + groups[-1]]
|
|
118
|
+
return [{"n": len(g), "p": _round4(sum(p for p, _ in g) / len(g)),
|
|
119
|
+
"right": _round4(sum(h for _, h in g) / len(g))} for g in groups]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _decide(d: Dist, tau: dict[str, Any]) -> str | None:
|
|
123
|
+
"""The top answer when it reaches its label's tau, as core.decide; None is unknown."""
|
|
124
|
+
top = top_label(d)
|
|
125
|
+
return top if tau.get(top) is not None and d[top] >= tau[top] else None
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _label_metrics(lab: str, rep: list[int], top: list[str], gold: dict[int, str],
|
|
129
|
+
passed: list[int], w: dict[int, float],
|
|
130
|
+
tau: float | None) -> dict[str, Any]:
|
|
131
|
+
"""One label's row of the eval record, over the reported items `rep`; `passed`: those at or
|
|
132
|
+
above their tau; w: each item's weight. tau stays exact: rounded up, it would refuse the very
|
|
133
|
+
units at the cut."""
|
|
134
|
+
tp = sum(top[i] == lab == gold[i] for i in rep)
|
|
135
|
+
pred = sum(top[i] == lab for i in rep)
|
|
136
|
+
true = sum(gold[i] == lab for i in rep)
|
|
137
|
+
p = tp / pred if pred else None
|
|
138
|
+
r = tp / true if true else None
|
|
139
|
+
ok = [i for i in passed if top[i] == lab]
|
|
140
|
+
hits = sum(gold[i] == lab for i in ok)
|
|
141
|
+
wk, wa = sum(w[i] for i in passed), sum(w.values())
|
|
142
|
+
return {"n": true, "precision": _round4(p), "recall": _round4(r),
|
|
143
|
+
"f1": _round4(2 * p * r / (p + r)) if p and r else (0.0 if pred or true else None),
|
|
144
|
+
"tau": tau, "coverage": _round4(len(ok) / pred) if pred else None,
|
|
145
|
+
"precision_at_tau": _round4(hits / len(ok)) if ok else None,
|
|
146
|
+
"precision_lb": _round4(wilson_lb(hits, len(ok))),
|
|
147
|
+
"share_error_pp": _round4(abs(sum(w[i] for i in ok) / wk
|
|
148
|
+
- sum(w[i] for i in rep if gold[i] == lab) / wa)
|
|
149
|
+
* 100) if wk else None}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _per_label(preds: Sequence[str | None], golds: list[str],
|
|
153
|
+
labels: list[str]) -> dict[str, dict[str, Any]]:
|
|
154
|
+
out = {}
|
|
155
|
+
for lab in labels:
|
|
156
|
+
tp = sum(p == lab == g for p, g in zip(preds, golds))
|
|
157
|
+
pred, true = sum(p == lab for p in preds), sum(g == lab for g in golds)
|
|
158
|
+
out[lab] = {"n": true, "precision": _round4(tp / pred) if pred else None,
|
|
159
|
+
"recall": _round4(tp / true) if true else None,
|
|
160
|
+
"f1": _round4(2 * tp / (pred + true)) if pred or true else None}
|
|
161
|
+
return out
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _macro(per: dict[str, dict[str, Any]]) -> float | None:
|
|
165
|
+
f1s = [v["f1"] for v in per.values() if v["f1"] is not None]
|
|
166
|
+
return _round4(sum(f1s) / len(f1s)) if f1s else None
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _report(raws: list[Dist], dists: list[Dist], preds: Sequence[str | None], golds: list[str],
|
|
170
|
+
labels: list[str]) -> dict[str, Any]:
|
|
171
|
+
"""The numbers of one report: at the tuned tau, without a threshold, and off the shelf (raw
|
|
172
|
+
scores at the default threshold of 0.5)."""
|
|
173
|
+
tops = [top_label(d) for d in dists]
|
|
174
|
+
plain, tuned = _per_label(tops, golds, labels), _per_label(preds, golds, labels)
|
|
175
|
+
half = dict.fromkeys(labels, 0.5)
|
|
176
|
+
default = _per_label([_decide(d, half) for d in raws], golds, labels)
|
|
177
|
+
rows = list(zip(dists, golds))
|
|
178
|
+
return {"n": len(golds), "macro_f1": _macro(plain), "macro_f1_tuned": _macro(tuned),
|
|
179
|
+
"macro_f1_default": _macro(default), "kappa": _round4(kappa(list(zip(tops, golds)))),
|
|
180
|
+
"brier": _round4(brier(rows)), "nll": _round4(nll(rows)),
|
|
181
|
+
"coverage": _round4(sum(p is not None for p in preds) / len(preds)),
|
|
182
|
+
"per_label": {lab: {**plain[lab], "precision_at_tau": tuned[lab]["precision"],
|
|
183
|
+
"recall_at_tau": tuned[lab]["recall"],
|
|
184
|
+
"f1_at_tau": tuned[lab]["f1"],
|
|
185
|
+
"precision_default": default[lab]["precision"],
|
|
186
|
+
"recall_default": default[lab]["recall"]} for lab in labels}}
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _score(backend: Backend, reqs: Sequence[Request],
|
|
190
|
+
prev: dict[str, Any] | None = None) -> tuple[dict[str, Dist], dict[str, Any]]:
|
|
191
|
+
"""(unit_id -> raw distribution, speed): saved scores, and the model only for units without
|
|
192
|
+
one. model_rev must be final by now, so run its prepare() first. The first call is the warm-up
|
|
193
|
+
(load plus one decision), timed apart; when this run timed no more calls than `prev` did,
|
|
194
|
+
prev's speed stands. Only a score of the same complete request can be reused."""
|
|
195
|
+
p = scores_path(reqs[0].task, backend.model_id, backend.model_rev, reqs[0].prompt_ver,
|
|
196
|
+
getattr(backend, "protocol", ""))
|
|
197
|
+
cached = {(r["unit_id"], r.get("input_hash")): r["dist"] for r in read_jsonl(p)}
|
|
198
|
+
saved: dict[str, Dist] = {}
|
|
199
|
+
times = []
|
|
200
|
+
for r in reqs:
|
|
201
|
+
digest = hashlib.sha256(json.dumps(asdict(r), sort_keys=True,
|
|
202
|
+
allow_nan=False).encode()).hexdigest()
|
|
203
|
+
if (r.unit_id, digest) in cached:
|
|
204
|
+
saved[r.unit_id] = cached[r.unit_id, digest]
|
|
205
|
+
else:
|
|
206
|
+
t0 = time.monotonic()
|
|
207
|
+
saved[r.unit_id] = backend.distribution(r)
|
|
208
|
+
times.append(time.monotonic() - t0)
|
|
209
|
+
append_jsonl(p, {"unit_id": r.unit_id, "input_hash": digest, "dist": saved[r.unit_id]})
|
|
210
|
+
warm = times[1:]
|
|
211
|
+
if prev and prev.get("n_timed", 0) >= len(warm):
|
|
212
|
+
return saved, {k: prev.get(k) for k in ("s_per_decision", "load_s", "peak_mb", "n_timed")}
|
|
213
|
+
peak = getattr(backend, "peak_memory_mb", None)
|
|
214
|
+
return saved, {"s_per_decision": _round4(sum(warm) / len(warm)) if warm else None,
|
|
215
|
+
"load_s": _round4(times[0]) if times else None, "n_timed": len(warm),
|
|
216
|
+
"peak_mb": peak() if peak and times else None}
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _fit_calibration(kind: str, raws: list[Dist], golds: list[str], yes: str) -> dict[str, Any]:
|
|
220
|
+
"""Keyword arguments of core.calibrate for one way to calibrate, fitted on these units."""
|
|
221
|
+
if kind == "temperature":
|
|
222
|
+
return {"temperature": fit_temperature(raws, golds), "platt": None}
|
|
223
|
+
return {"temperature": 1.0, "platt": fit_platt(raws, golds, yes) if kind == "platt" else None}
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def evaluate(task: str, backend: Backend, items: Sequence[_Item], precision: float, recall: float,
|
|
227
|
+
today: date, prev: dict[str, Any] | None = None) -> dict[str, Any] | None:
|
|
228
|
+
"""Items never come from a final test day. The eval record of one backend (numbers), out of
|
|
229
|
+
fold; None when nothing is left to report on."""
|
|
230
|
+
labels = [c for c, _d in TASKS[task].candidates]
|
|
231
|
+
# the encoder's training days never calibrate it
|
|
232
|
+
items = [it for it in items
|
|
233
|
+
if not (isinstance(backend, Encoder) and in_encoder_train_split(it.req))]
|
|
234
|
+
if not items:
|
|
235
|
+
return None
|
|
236
|
+
reqs = [it.req for it in items]
|
|
237
|
+
golds = [it.gold for it in items]
|
|
238
|
+
w = [it.weight for it in items]
|
|
239
|
+
saved, speed = _score(backend, reqs, prev)
|
|
240
|
+
raws = [{lab: saved[r.unit_id].get(lab, 0.0) for lab in labels} for r in reqs]
|
|
241
|
+
folds = [fold(r) for r in reqs]
|
|
242
|
+
every = range(len(reqs))
|
|
243
|
+
split = [([i for i in every if folds[i] != k], [i for i in every if folds[i] == k])
|
|
244
|
+
for k in range(FOLDS)]
|
|
245
|
+
|
|
246
|
+
def fit(kind: str, on: Iterable[int]) -> dict[str, Any]:
|
|
247
|
+
on = list(on)
|
|
248
|
+
return _fit_calibration(kind, [raws[i] for i in on], [golds[i] for i in on], labels[0])
|
|
249
|
+
|
|
250
|
+
def out_of_fold(kind: str) -> list[Dist]:
|
|
251
|
+
out: list[Dist] = [{}] * len(reqs)
|
|
252
|
+
for train, test in split:
|
|
253
|
+
c = fit(kind, train)
|
|
254
|
+
for i in test:
|
|
255
|
+
out[i] = calibrate(raws[i], **c)
|
|
256
|
+
return out
|
|
257
|
+
|
|
258
|
+
# T = 1 unless a fit lowers the out-of-fold Brier. shortcut: the way to calibrate is picked on
|
|
259
|
+
# the folds it reports on (one pick of at most three), upgrade by nesting it if gold grows.
|
|
260
|
+
kinds = ("none", "temperature") + (("platt",) if TASKS[task].decision == "boolean" else ())
|
|
261
|
+
briers = {k: _round4(brier(list(zip(out_of_fold(k), golds)))) or 0.0 for k in kinds}
|
|
262
|
+
kind = min(kinds, key=briers.__getitem__)
|
|
263
|
+
dists = out_of_fold(kind)
|
|
264
|
+
preds: list[str | None] = [None] * len(reqs)
|
|
265
|
+
no_threshold: dict[str, list[str]] = {}
|
|
266
|
+
for k, (train, test) in enumerate(split):
|
|
267
|
+
c = fit(kind, train)
|
|
268
|
+
tau = fit_tau([(top_label(d), d[top_label(d)], golds[i]) for i in train
|
|
269
|
+
for d in [calibrate(raws[i], **c)]], precision, recall=recall)
|
|
270
|
+
missing = sorted(lab for lab in set(golds) if tau.get(lab) is None)
|
|
271
|
+
if test and missing:
|
|
272
|
+
no_threshold[str(k)] = missing
|
|
273
|
+
for i in test:
|
|
274
|
+
preds[i] = _decide(dists[i], tau)
|
|
275
|
+
saved_cal = fit(kind, every)
|
|
276
|
+
full = [calibrate(r, **saved_cal) for r in raws]
|
|
277
|
+
tau = fit_tau([(top_label(d), d[top_label(d)], g) for d, g in zip(full, golds)], precision,
|
|
278
|
+
recall=recall)
|
|
279
|
+
rep = _report(raws, dists, preds, golds, labels)
|
|
280
|
+
tops = [top_label(d) for d in dists]
|
|
281
|
+
passed = [i for i in every if preds[i]]
|
|
282
|
+
for lab, v in rep["per_label"].items():
|
|
283
|
+
v.update(_label_metrics(lab, list(every), tops, dict(enumerate(golds)), passed,
|
|
284
|
+
dict(enumerate(w)), tau.get(lab)))
|
|
285
|
+
ver = TASKS[task].ver
|
|
286
|
+
return {"task": task, "task_ver": ver, "backend": backend.backend,
|
|
287
|
+
"model_id": backend.model_id, "model_rev": backend.model_rev,
|
|
288
|
+
"size_mb": backend.size_mb, "prompt_ver": reqs[0].prompt_ver,
|
|
289
|
+
"calib_ver": f"{task}@{ver}/{today.isoformat()}", "calibration": kind, **saved_cal,
|
|
290
|
+
"brier_by_calibration": briers, "n_cal": len(reqs), "n_held": len(reqs),
|
|
291
|
+
"folds": {str(k): len(test) for k, (_t, test) in enumerate(split)},
|
|
292
|
+
"no_threshold_folds": no_threshold, **rep, **speed,
|
|
293
|
+
"ece_raw": _round4(ece([(max(r.values()), top_label(r) == g)
|
|
294
|
+
for r, g in zip(raws, golds)])),
|
|
295
|
+
"ece_cal": _round4(ece([(max(d.values()), top_label(d) == g)
|
|
296
|
+
for d, g in zip(dists, golds)])),
|
|
297
|
+
"reliability_raw": reliability([(max(r.values()), top_label(r) == g)
|
|
298
|
+
for r, g in zip(raws, golds)]),
|
|
299
|
+
"reliability_cal": reliability([(max(d.values()), top_label(d) == g)
|
|
300
|
+
for d, g in zip(dists, golds)])}
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _gold_units(cfg: Config, task: str, days: int, src: Source | None,
|
|
304
|
+
today: date | None = None) -> list[tuple[Request, str]]:
|
|
305
|
+
"""(request, gold) for every gold unit still in the window, final test days included: each
|
|
306
|
+
caller takes its side of evalset.final_test."""
|
|
307
|
+
gold = gold_latest(task)
|
|
308
|
+
reqs = window_units(cfg, task, days, src, today)
|
|
309
|
+
found = [u for u in gold if u in reqs]
|
|
310
|
+
if len(found) < len(gold):
|
|
311
|
+
print(f"aut: label eval: {len(gold) - len(found)} gold units are outside the {days}-day "
|
|
312
|
+
"window; left out", file=sys.stderr)
|
|
313
|
+
return [(reqs[u], gold[u]) for u in found]
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _warn_thin(task: str, golds: list[str], what: str) -> None:
|
|
317
|
+
floor = FLOORS[task][0]
|
|
318
|
+
n = Counter(golds)
|
|
319
|
+
thin = {c: n[c] for c, _d in TASKS[task].candidates if n[c] < floor}
|
|
320
|
+
if thin:
|
|
321
|
+
print(f"aut: label eval: {what} is too small to mean much: under {floor} gold per label "
|
|
322
|
+
f"for {json.dumps(thin)}", file=sys.stderr)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def default_backends(cfg: Config, task: str) -> list[Backend]:
|
|
326
|
+
"""Every model in the table (and label_model) that is fully on disk, plus the encoder once
|
|
327
|
+
trained. The others are printed with what it takes to fetch them."""
|
|
328
|
+
out: list[Backend] = []
|
|
329
|
+
for name in dict.fromkeys(list(MODELS) + [short_name(cfg.label_model)]):
|
|
330
|
+
b = make_backend(task, Choice(model_id=name), cfg.label_model, cfg.label_llama_server)
|
|
331
|
+
try:
|
|
332
|
+
getattr(b, "prepare", lambda: None)()
|
|
333
|
+
except BackendUnavailable as exc:
|
|
334
|
+
print(f"aut: label eval: {name} left out: {exc}", file=sys.stderr)
|
|
335
|
+
continue
|
|
336
|
+
out.append(b)
|
|
337
|
+
if encoder_path(task).exists():
|
|
338
|
+
out.append(Encoder(task))
|
|
339
|
+
return out
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _key(rec: dict[str, Any]) -> tuple[Any, ...]:
|
|
343
|
+
return rec.get("model_id"), rec.get("model_rev"), rec.get("prompt_ver")
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def newest(task: str) -> dict[tuple[Any, ...], dict[str, Any]]:
|
|
347
|
+
"""The newest record per model, revision and prompt of the current task version, the most
|
|
348
|
+
recent one last. Records from before the five folds have no Brier and do not count."""
|
|
349
|
+
out: dict[tuple[Any, ...], dict[str, Any]] = {}
|
|
350
|
+
for r in read_jsonl(eval_path(task, "eval")):
|
|
351
|
+
if r.get("task_ver") == TASKS[task].ver and "brier" in r:
|
|
352
|
+
out.pop(_key(r), None)
|
|
353
|
+
out[_key(r)] = r
|
|
354
|
+
return out
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def run_eval(cfg: Config, task: str, days: int = LABEL_DAYS, *, src: Source | None = None,
|
|
358
|
+
backends: list[Backend] | None = None, now: datetime | None = None) -> list[dict[str, Any]]:
|
|
359
|
+
now = now or datetime.now(timezone.utc)
|
|
360
|
+
today = now.astimezone().date()
|
|
361
|
+
pool = [(r, g) for r, g in _gold_units(cfg, task, days, src, today) if not final_test(r)]
|
|
362
|
+
if not pool:
|
|
363
|
+
raise AutError(f"label eval: no gold labels for {task} in the last {days} days outside "
|
|
364
|
+
"the final test days (label some with `aut label review`)")
|
|
365
|
+
_warn_thin(task, [g for _q, g in pool], "the tuning gold")
|
|
366
|
+
w = weights(task, [r.unit_id for r, _g in pool])
|
|
367
|
+
items = [_Item(r, g, w[r.unit_id]) for r, g in pool]
|
|
368
|
+
before, recs = newest(task), []
|
|
369
|
+
for b in backends if backends is not None else default_backends(cfg, task):
|
|
370
|
+
try:
|
|
371
|
+
getattr(b, "prepare", lambda: None)()
|
|
372
|
+
rec = evaluate(task, b, items, cfg.label_precision, cfg.label_recall, today,
|
|
373
|
+
before.get((b.model_id, b.model_rev, items[0].req.prompt_ver)))
|
|
374
|
+
except BackendUnavailable as exc: # one model that cannot run here: the others still run
|
|
375
|
+
print(f"aut: label eval: {b.model_id} left out: {exc}", file=sys.stderr)
|
|
376
|
+
continue
|
|
377
|
+
finally:
|
|
378
|
+
getattr(b, "close", lambda: None)()
|
|
379
|
+
if rec is None:
|
|
380
|
+
print(f"aut: label eval: {b.model_id} has no gold outside its training days; left out",
|
|
381
|
+
file=sys.stderr)
|
|
382
|
+
continue
|
|
383
|
+
append_jsonl(eval_path(task, "eval"), {"ts": now.isoformat(), **rec})
|
|
384
|
+
if cfg.pg_dsn:
|
|
385
|
+
send_activity_rows(cfg.pg_dsn, [ActivityEvent(
|
|
386
|
+
event_id=make_event_id("eval", json.dumps(rec, sort_keys=True)),
|
|
387
|
+
ts=now.isoformat(), agent="label", kind="eval", session_id=None, turn_id=None,
|
|
388
|
+
attrs=rec, machine=cfg.machine)])
|
|
389
|
+
recs.append(rec)
|
|
390
|
+
return recs
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def run_final(cfg: Config, task: str, model: str, days: int = LABEL_DAYS, *,
|
|
394
|
+
src: Source | None = None, backend: Backend | None = None,
|
|
395
|
+
now: datetime | None = None) -> dict[str, Any]:
|
|
396
|
+
"""Score one model's saved calibration and tau on the final test days. Nothing fits here, and
|
|
397
|
+
only this function reads those days; the result goes to its own file, which nothing reads."""
|
|
398
|
+
now = now or datetime.now(timezone.utc)
|
|
399
|
+
rec = _saved_result(task, model)
|
|
400
|
+
pairs = [(r, g) for r, g in _gold_units(cfg, task, days, src, now.astimezone().date())
|
|
401
|
+
if final_test(r)]
|
|
402
|
+
if not pairs:
|
|
403
|
+
raise AutError(f"label eval --final: no gold labels for {task} on the final test days "
|
|
404
|
+
"yet (label more days with `aut label review`)")
|
|
405
|
+
reqs, golds = [r for r, _g in pairs], [g for _r, g in pairs]
|
|
406
|
+
if rec["prompt_ver"] != reqs[0].prompt_ver:
|
|
407
|
+
raise AutError(f"label eval --final: {model} was tuned on prompt {rec['prompt_ver']}; "
|
|
408
|
+
"run `aut label eval` again first")
|
|
409
|
+
_warn_thin(task, golds, "the final test set")
|
|
410
|
+
out = eval_path(task, "final")
|
|
411
|
+
looks = sum(r.get("model_id") == rec["model_id"] for r in read_jsonl(out))
|
|
412
|
+
if looks:
|
|
413
|
+
print(f"aut: label eval: {rec['model_id']} has seen the final test set {looks} times; "
|
|
414
|
+
"if you changed anything because of it, it no longer tests that", file=sys.stderr)
|
|
415
|
+
b = backend or make_backend(task, Choice(backend=rec["backend"], model_id=rec["model_id"],
|
|
416
|
+
model_rev=rec["model_rev"]), cfg.label_model,
|
|
417
|
+
cfg.label_llama_server)
|
|
418
|
+
try:
|
|
419
|
+
getattr(b, "prepare", lambda: None)()
|
|
420
|
+
saved, _speed = _score(b, reqs)
|
|
421
|
+
finally:
|
|
422
|
+
getattr(b, "close", lambda: None)()
|
|
423
|
+
labels = [c for c, _d in TASKS[task].candidates]
|
|
424
|
+
raws = [{lab: saved[r.unit_id].get(lab, 0.0) for lab in labels} for r in reqs]
|
|
425
|
+
tau = {k: v["tau"] for k, v in rec["per_label"].items()}
|
|
426
|
+
dists = [calibrate(r, rec["temperature"], rec.get("platt")) for r in raws]
|
|
427
|
+
res = {"ts": now.isoformat(), "task": task, "task_ver": rec["task_ver"],
|
|
428
|
+
"backend": rec["backend"], "model_id": rec["model_id"], "model_rev": rec["model_rev"],
|
|
429
|
+
"prompt_ver": rec["prompt_ver"], "calib_ver": rec["calib_ver"],
|
|
430
|
+
**_report(raws, dists, [_decide(d, tau) for d in dists], golds, labels)}
|
|
431
|
+
append_jsonl(out, res)
|
|
432
|
+
res["fails"] = failures(res, cfg.label_precision, cfg.label_recall)
|
|
433
|
+
return res
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def final_line(r: dict[str, Any]) -> str:
|
|
437
|
+
fails = "; ".join(r["fails"]) or "none"
|
|
438
|
+
return (f"label eval --final {r['task']} {r['model_id']}: n {r['n']}, macro-F1 tuned "
|
|
439
|
+
f"{r['macro_f1_tuned']} (default 0.5: {r['macro_f1_default']}), Brier {r['brier']}, "
|
|
440
|
+
f"NLL {r['nll']}, coverage {r['coverage']}, floors missed: {fails}")
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def failures(rec: dict[str, Any], precision: float, recall: float,
|
|
444
|
+
coverage: float | None = None) -> list[str]:
|
|
445
|
+
"""Why a report misses the bar: a label under the precision or recall floor at its tau (a
|
|
446
|
+
label with no gold has no recall to judge), and, when coverage is given, the rest of the bar
|
|
447
|
+
(CONTRACT.md, the bake-off): coverage at tau, and no label's share on known items off by over
|
|
448
|
+
SHARE_ERROR_PP."""
|
|
449
|
+
out = []
|
|
450
|
+
for lab, v in rec["per_label"].items():
|
|
451
|
+
if v["precision_at_tau"] is not None and v["precision_at_tau"] < precision:
|
|
452
|
+
out.append(f"{lab} precision {v['precision_at_tau']}")
|
|
453
|
+
if v["recall_at_tau"] is not None and v["recall_at_tau"] < recall:
|
|
454
|
+
out.append(f"{lab} recall {v['recall_at_tau']}")
|
|
455
|
+
if coverage is not None and (v["share_error_pp"] or 0) > SHARE_ERROR_PP:
|
|
456
|
+
out.append(f"{lab} share error {v['share_error_pp']}")
|
|
457
|
+
if coverage is not None and (rec["coverage"] or 0) < coverage:
|
|
458
|
+
out.append(f"coverage {rec['coverage']}")
|
|
459
|
+
return out
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _fails(rec: dict[str, Any], cfg: Config) -> list[str]:
|
|
463
|
+
return failures(rec, cfg.label_precision, cfg.label_recall, cfg.label_coverage)
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def pareto(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
467
|
+
"""The rows no other row beats on quality (tuned macro-F1), speed and memory. An axis a row
|
|
468
|
+
does not know (null) is not compared."""
|
|
469
|
+
def axes(r: dict[str, Any]) -> tuple[Any, ...]:
|
|
470
|
+
return -(r.get("macro_f1_tuned") or 0), r.get("s_per_decision"), r.get("peak_mb")
|
|
471
|
+
|
|
472
|
+
def beats(a: dict[str, Any], b: dict[str, Any]) -> bool:
|
|
473
|
+
pairs = [(x, y) for x, y in zip(axes(a), axes(b)) if x is not None and y is not None]
|
|
474
|
+
return all(x <= y for x, y in pairs) and any(x < y for x, y in pairs)
|
|
475
|
+
return [r for r in rows if not any(beats(o, r) for o in rows)]
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def recommend(rows: list[dict[str, Any]], cfg: Config) -> dict[str, Any] | None:
|
|
479
|
+
"""Of the frontier of models that pass: the best quality (to two decimals: closer is noise on
|
|
480
|
+
gold this size), then the fastest."""
|
|
481
|
+
front = pareto([r for r in rows if not _fails(r, cfg)])
|
|
482
|
+
return min(front, key=lambda r: (-round(r.get("macro_f1_tuned") or 0, 2),
|
|
483
|
+
r.get("s_per_decision") or math.inf,
|
|
484
|
+
r.get("peak_mb") or math.inf), default=None)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def table(cfg: Config, task: str) -> list[str]:
|
|
488
|
+
"""Quality, speed and memory per model. `*` on the Pareto frontier of the models that pass,
|
|
489
|
+
`!` fails a floor or the bar. Numbers are out of fold; none comes from the final test days."""
|
|
490
|
+
rows = sorted(newest(task).values(), key=lambda r: -(r.get("macro_f1_tuned") or 0))
|
|
491
|
+
front = pareto([r for r in rows if not _fails(r, cfg)])
|
|
492
|
+
lines = [f"label eval {task}: floors precision {cfg.label_precision}, recall "
|
|
493
|
+
f"{cfg.label_recall}, coverage {cfg.label_coverage}",
|
|
494
|
+
f" {'model':<46} {'rev':<8} {'F1':>6} {'F1@.5':>6} {'Brier':>6} {'NLL':>6} "
|
|
495
|
+
f"{'calib':<11} {'s/dec':>7} {'load s':>7} {'peak MB':>7} {'n':>5}"]
|
|
496
|
+
for r in rows:
|
|
497
|
+
mark = ("*" if any(r is o for o in front) else " ") + ("!" if _fails(r, cfg) else " ")
|
|
498
|
+
lines.append(f"{mark} {short_name(r['model_id']):<46} {r['model_rev'][:8]:<8} "
|
|
499
|
+
f"{r['macro_f1_tuned']!s:>6} {r['macro_f1_default']!s:>6} {r['brier']!s:>6} "
|
|
500
|
+
f"{r['nll']!s:>6} {r['calibration']:<11} {r.get('s_per_decision')!s:>7} "
|
|
501
|
+
f"{r.get('load_s')!s:>7} {r.get('peak_mb')!s:>7} {r['n']:>5}")
|
|
502
|
+
return lines + ["* Pareto frontier of the models that pass ! fails: "
|
|
503
|
+
+ "; ".join(f"{short_name(r['model_id'])}: {', '.join(_fails(r, cfg))}"
|
|
504
|
+
for r in rows if _fails(r, cfg))]
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def details(recs: list[dict[str, Any]]) -> list[str]:
|
|
508
|
+
"""Per label and reliability of the models just evaluated."""
|
|
509
|
+
def rel(rows: list[dict[str, Any]]) -> str:
|
|
510
|
+
return " ".join(f"{g['p']}:{g['right']}" for g in rows)
|
|
511
|
+
|
|
512
|
+
out = []
|
|
513
|
+
for r in recs:
|
|
514
|
+
out += ["", f"{r['model_id']}: {r['calibration']} (T {r['temperature']}, platt "
|
|
515
|
+
f"{json.dumps(r['platt'])}), Brier {r['brier_by_calibration']}, folds "
|
|
516
|
+
f"{r['folds']}, no threshold in folds {r['no_threshold_folds'] or 'none'}",
|
|
517
|
+
f" {'label':<16} {'n':>4} {'tau':>6} {'P':>6} {'R':>6} "
|
|
518
|
+
f"at 0.5: {'P':>6} {'R':>6}"]
|
|
519
|
+
out += [f" {lab:<16} {v['n']:>4} {_round4(v['tau'])!s:>6} {v['precision_at_tau']!s:>6} "
|
|
520
|
+
f"{v['recall_at_tau']!s:>6} {v['precision_default']!s:>6} "
|
|
521
|
+
f"{v['recall_default']!s:>6}" for lab, v in r["per_label"].items()]
|
|
522
|
+
out += [f" reliability raw (mean p:share right) {rel(r['reliability_raw'])}",
|
|
523
|
+
f" reliability calibrated {rel(r['reliability_cal'])}"]
|
|
524
|
+
return out
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def _saved_result(task: str, model: str) -> dict[str, Any]:
|
|
528
|
+
"""The newest record of a model id (or a short name from backends.MODELS)."""
|
|
529
|
+
known = find_model(model)
|
|
530
|
+
repo = known.repo if known else model
|
|
531
|
+
match = [r for r in newest(task).values() if r["model_id"] == repo]
|
|
532
|
+
if not match:
|
|
533
|
+
raise AutError(f"label: no eval record of {model} for {task}; run `aut label eval` "
|
|
534
|
+
f"first. Known: {sorted({r['model_id'] for r in newest(task).values()})}")
|
|
535
|
+
return match[-1]
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def choose(cfg: Config, task: str, model: str | None = None) -> dict[str, Any] | None:
|
|
539
|
+
"""With a model: write its saved result into label.json; other tasks untouched. Without one:
|
|
540
|
+
print the table and a recommendation, and write nothing."""
|
|
541
|
+
if model is None:
|
|
542
|
+
rows = list(newest(task).values())
|
|
543
|
+
if not rows:
|
|
544
|
+
raise AutError(f"label choose: no eval record for {task}; run `aut label eval` first")
|
|
545
|
+
rec = recommend(rows, cfg)
|
|
546
|
+
print("\n".join(table(cfg, task)))
|
|
547
|
+
print(f"recommended: aut label choose --task {task} --model "
|
|
548
|
+
f"{short_name(rec['model_id'])}" if rec else
|
|
549
|
+
"recommended: none, no model passes the floors. They stay as they are: label more "
|
|
550
|
+
"gold, or try other models.")
|
|
551
|
+
return None
|
|
552
|
+
win = _saved_result(task, model)
|
|
553
|
+
fails = _fails(win, cfg)
|
|
554
|
+
if fails:
|
|
555
|
+
print(f"aut: warning: {win['model_id']} fails the floors for {task}: {', '.join(fails)}. "
|
|
556
|
+
"Labels without a tau stay unknown.", file=sys.stderr)
|
|
557
|
+
p = choice_path()
|
|
558
|
+
try:
|
|
559
|
+
obj = read_choices()
|
|
560
|
+
except (OSError, ValueError) as exc: # overwriting would lose every other task's choice
|
|
561
|
+
raise AutError(f"label choose: {p} is unusable ({exc}); nothing written. Fix or move "
|
|
562
|
+
"it aside and run choose again for each task.") from None
|
|
563
|
+
obj.setdefault("tasks", {})[task] = {
|
|
564
|
+
"backend": win["backend"], "model_id": win["model_id"], "model_rev": win["model_rev"],
|
|
565
|
+
"temperature": win["temperature"], "platt": win.get("platt"),
|
|
566
|
+
"calib_ver": win["calib_ver"], "prompt_ver": win["prompt_ver"],
|
|
567
|
+
"tau": {k: v["tau"] for k, v in win["per_label"].items() if v["tau"] is not None}}
|
|
568
|
+
atomic_write_text(p, json.dumps(obj, indent=2, sort_keys=True))
|
|
569
|
+
return obj["tasks"][task]
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
MIN_UPDATES = 2000
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def _fit(data: list[tuple[list[int], list[float], float]], k: int, epochs: int = 12,
|
|
576
|
+
lr: float = 0.3, l2: float = 1e-4, seed: int = 0) -> dict[int, list[float]]:
|
|
577
|
+
"""Multinomial logistic regression by SGD on (features, target distribution, weight)."""
|
|
578
|
+
w: dict[int, list[float]] = {}
|
|
579
|
+
rng, data = random.Random(seed), list(data) # nosec B311 # SGD shuffle order, not security
|
|
580
|
+
# a small gold set gets more passes: 12 passes over 20 items underfit even a separable set
|
|
581
|
+
epochs = max(epochs, -(-MIN_UPDATES // max(1, len(data))))
|
|
582
|
+
for ep in range(epochs):
|
|
583
|
+
rng.shuffle(data)
|
|
584
|
+
step = lr / (1 + ep)
|
|
585
|
+
for feats, target, wt in data:
|
|
586
|
+
rows = [w.setdefault(f, [0.0] * k) for f in feats]
|
|
587
|
+
p = softmax([sum(r[j] for r in rows) for j in range(k)])
|
|
588
|
+
g = [(p[j] - target[j]) * wt * step for j in range(k)]
|
|
589
|
+
for r in rows:
|
|
590
|
+
for j in range(k):
|
|
591
|
+
r[j] -= g[j] + step * l2 * r[j]
|
|
592
|
+
return w
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def _training_data(gold: dict[str, str], silver: dict[str, dict[str, Any]], train: dict[str, Request],
|
|
596
|
+
labels: list[str]) -> list[tuple[list[int], list[float], float]]:
|
|
597
|
+
"""(features, target distribution, weight) of each gold and silver unit of the training days."""
|
|
598
|
+
data = []
|
|
599
|
+
for u, g in gold.items():
|
|
600
|
+
if u in train:
|
|
601
|
+
data.append((features(train[u].facts), [float(x == g) for x in labels], GOLD_WEIGHT))
|
|
602
|
+
for u, ts in silver.items():
|
|
603
|
+
if u in train and u not in gold and ts:
|
|
604
|
+
avg = [sum(d.get(x, 0.0) for d in ts.values()) / len(ts) for x in labels]
|
|
605
|
+
data.append((features(train[u].facts), avg, 1.0))
|
|
606
|
+
return data
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
def _macro_f1(pairs: list[tuple[str, str]], labels: list[str]) -> float | None:
|
|
610
|
+
f1s = []
|
|
611
|
+
for lab in labels:
|
|
612
|
+
tp = sum(a == lab == b for a, b in pairs)
|
|
613
|
+
pred, true = sum(a == lab for a, _ in pairs), sum(b == lab for _, b in pairs)
|
|
614
|
+
if pred or true:
|
|
615
|
+
f1s.append(2 * tp / (pred + true))
|
|
616
|
+
return _round4(sum(f1s) / len(f1s)) if f1s else None
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
def train_encoder(cfg: Config, task: str, days: int = LABEL_DAYS, *, src: Source | None = None,
|
|
620
|
+
now: datetime | None = None) -> dict[str, Any]:
|
|
621
|
+
"""Fit the encoder on the gold (GOLD_WEIGHT) and silver (teachers averaged) of its training
|
|
622
|
+
days (in_encoder_train_split), never on a final test day nor on a day eval calibrates it with;
|
|
623
|
+
report accuracy and macro-F1 on the gold of those other days."""
|
|
624
|
+
labels = [c for c, _d in TASKS[task].candidates]
|
|
625
|
+
today = (now or datetime.now(timezone.utc)).astimezone().date()
|
|
626
|
+
reqs = window_units(cfg, task, days, src, today)
|
|
627
|
+
gold, silver = gold_latest(task), silver_by_unit(task)
|
|
628
|
+
open_days = {u: r for u, r in reqs.items() if not final_test(r)}
|
|
629
|
+
train = {u: r for u, r in open_days.items() if in_encoder_train_split(r)}
|
|
630
|
+
test = {u for u in gold if u in open_days and u not in train}
|
|
631
|
+
data = _training_data(gold, silver, train, labels)
|
|
632
|
+
if not data:
|
|
633
|
+
raise AutError(f"label train-encoder: no gold or silver labels for {task} yet")
|
|
634
|
+
w = _fit(data, len(labels))
|
|
635
|
+
# atomic: a crash mid-write must not leave a corrupt encoder that the nightly run then loads
|
|
636
|
+
atomic_write_text(encoder_path(task), json.dumps(
|
|
637
|
+
{"task": task, "task_ver": TASKS[task].ver, "labels": labels,
|
|
638
|
+
"w": {str(f): [round(x, 5) for x in r] for f, r in w.items()}}))
|
|
639
|
+
enc = Encoder(task)
|
|
640
|
+
pairs = [(top_label(enc.distribution(reqs[u])), gold[u]) for u in sorted(test)]
|
|
641
|
+
return {"model_id": enc.model_id, "model_rev": enc.model_rev, "trained_on": len(data),
|
|
642
|
+
"held_out": len(pairs),
|
|
643
|
+
"accuracy": _round4(sum(a == b for a, b in pairs) / len(pairs)) if pairs else None,
|
|
644
|
+
"macro_f1": _macro_f1(pairs, labels)}
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
def print_eval(cfg: Config, task: str, records: list[dict[str, Any]]) -> None:
|
|
648
|
+
print("\n".join(table(cfg, task) + details(records)))
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def print_final(r: dict[str, Any]) -> None:
|
|
652
|
+
print(final_line(r))
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
def print_choose(task: str, c: dict[str, Any]) -> None:
|
|
656
|
+
print(f"label choose {task}: {c['backend']} {c['model_id']} (T {c['temperature']}, platt "
|
|
657
|
+
f"{json.dumps(c['platt'])}, tau {json.dumps(c['tau'])})")
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
def print_train_encoder(task: str, r: dict[str, Any]) -> None:
|
|
661
|
+
print(f"label train-encoder {task}: {json.dumps(r)}")
|