agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,661 @@
1
+ """`aut label eval`, `aut label choose` and `aut label train-encoder`: the bake-off on gold.
2
+
3
+ Raw scores are saved per task, model, revision and prompt, so a rerun calls the model only for new
4
+ gold units and tuning needs no model. Calibration and the per-label tau fit on four of five folds
5
+ (evalset.fold: same for every model, grouped by day) and report on the fifth; the saved calibration
6
+ and tau then fit on all five. A floor on precision and on recall per label is never relaxed: a
7
+ model that misses one is marked as failing. A stable part of the days (evalset.final_test) stays
8
+ out of all of this; only `aut label eval --final --model M` scores it, into <task>.final.jsonl.
9
+
10
+ Per backend, one eval record of numbers (and model ids) is appended to <task>.eval.jsonl and sent
11
+ as an activity_event of kind `eval`. `aut label eval` prints one table per task (quality, speed,
12
+ memory, with Pareto and failure marks); the owner picks with `aut label choose --task T --model M`,
13
+ which writes that model's saved result into label.json, which the nightly run reads.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import json
19
+ import hashlib
20
+ import math
21
+ import random
22
+ import sys
23
+ import time
24
+ from collections import Counter
25
+ from dataclasses import asdict
26
+ from datetime import date, datetime, timezone
27
+ from typing import Any, Iterable, NamedTuple, Sequence
28
+
29
+ from ..config import LABEL_DAYS, AutError, Config
30
+ from ..event import ActivityEvent, make_event_id
31
+ from ..push import send_activity_rows
32
+ from ..state import atomic_write_text
33
+ from .backends import (MODELS, BackendUnavailable, Encoder, encoder_path, features, find_model,
34
+ make_backend, short_name)
35
+ from .core import (Backend, Choice, Request, calibrate, choice_path, fit_platt, fit_tau,
36
+ fit_temperature, read_choices, softmax, top_label)
37
+ from .evalset import (FLOORS, FOLDS, append_jsonl, eval_path, final_test, fold, gold_latest,
38
+ in_encoder_train_split, read_jsonl, scores_path, silver_by_unit, stratum,
39
+ window_units)
40
+ from .tasks import TASKS, Source
41
+
42
+ SHARE_ERROR_PP = 5.0 # a label's share on known items may be off this much (CONTRACT.md)
43
+ GOLD_WEIGHT = 3.0 # train-encoder: one gold answer counts as three silver ones
44
+ Dist = dict[str, float]
45
+
46
+
47
+ class _Item(NamedTuple):
48
+ """One gold unit of a bake-off."""
49
+ req: Request
50
+ gold: str
51
+ weight: float
52
+
53
+
54
+ def _round4(x: float | None) -> float | None:
55
+ return None if x is None else round(x, 4)
56
+
57
+
58
+ def wilson_lb(k: int, n: int, z: float = 1.96) -> float | None:
59
+ """Lower bound of the 95% Wilson interval of k right out of n."""
60
+ if not n:
61
+ return None
62
+ p = k / n
63
+ return (p + z * z / (2 * n) - z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n))) \
64
+ / (1 + z * z / n)
65
+
66
+
67
+ def kappa(pairs: list[tuple[str, str]]) -> float | None:
68
+ """Cohen's kappa of (predicted, gold)."""
69
+ if not pairs:
70
+ return None
71
+ n = len(pairs)
72
+ po = sum(a == b for a, b in pairs) / n
73
+ pa, pb = Counter(a for a, _ in pairs), Counter(b for _, b in pairs)
74
+ pe = sum(pa[x] * pb[x] for x in pa) / (n * n)
75
+ return 1.0 if pe == 1 else (po - pe) / (1 - pe)
76
+
77
+
78
+ def ece(tops: list[tuple[float, bool]], bins: int = 10) -> float | None:
79
+ """Expected calibration error of (top probability, was right), equal-width bins."""
80
+ if not tops:
81
+ return None
82
+ by: dict[int, list[Any]] = {}
83
+ for p, hit in tops:
84
+ by.setdefault(min(int(p * bins), bins - 1), []).append((p, hit))
85
+ return sum(abs(sum(p for p, _ in b) / len(b) - sum(h for _, h in b) / len(b)) * len(b)
86
+ for b in by.values()) / len(tops)
87
+
88
+
89
+ def weights(task: str, golds: list[str]) -> dict[str, float]:
90
+ """unit_id -> weight that re-weights the stratified gold sample to natural frequency: the
91
+ share of its silver stratum among every silver unit over its share among the gold units.
92
+ shortcut: the silver sample stands in for the population (teach samples by day and unit
93
+ type, not by label); upgrade by counting nightly rows once a task runs nightly."""
94
+ silver = silver_by_unit(task)
95
+ pop = Counter(stratum(t) for t in silver.values())
96
+ sample = Counter(stratum(silver.get(u)) for u in golds)
97
+ tp, ts = sum(pop.values()), len(golds)
98
+ return {u: (pop[s] / tp) / (sample[s] / ts) if pop[s] else 1.0
99
+ for u in golds for s in [stratum(silver.get(u))]}
100
+
101
+
102
+ def brier(rows: list[tuple[Dist, str]]) -> float:
103
+ """Mean over (distribution, gold) of the squared error summed over candidates: 0 to 2."""
104
+ return sum(sum((p - (k == g)) ** 2 for k, p in d.items()) for d, g in rows) / len(rows)
105
+
106
+
107
+ def nll(rows: list[tuple[Dist, str]]) -> float:
108
+ return -sum(math.log(max(d[g], 1e-12)) for d, g in rows) / len(rows)
109
+
110
+
111
+ def reliability(tops: list[tuple[float, bool]], size: int = 20) -> list[dict[str, Any]]:
112
+ """(top probability, was right), lowest first, in groups of about `size`: mean p, share
113
+ right."""
114
+ tops = sorted(tops)
115
+ groups = [tops[i:i + size] for i in range(0, len(tops), size)]
116
+ if len(groups) > 1 and len(groups[-1]) < size / 2:
117
+ groups[-2:] = [groups[-2] + groups[-1]]
118
+ return [{"n": len(g), "p": _round4(sum(p for p, _ in g) / len(g)),
119
+ "right": _round4(sum(h for _, h in g) / len(g))} for g in groups]
120
+
121
+
122
+ def _decide(d: Dist, tau: dict[str, Any]) -> str | None:
123
+ """The top answer when it reaches its label's tau, as core.decide; None is unknown."""
124
+ top = top_label(d)
125
+ return top if tau.get(top) is not None and d[top] >= tau[top] else None
126
+
127
+
128
+ def _label_metrics(lab: str, rep: list[int], top: list[str], gold: dict[int, str],
129
+ passed: list[int], w: dict[int, float],
130
+ tau: float | None) -> dict[str, Any]:
131
+ """One label's row of the eval record, over the reported items `rep`; `passed`: those at or
132
+ above their tau; w: each item's weight. tau stays exact: rounded up, it would refuse the very
133
+ units at the cut."""
134
+ tp = sum(top[i] == lab == gold[i] for i in rep)
135
+ pred = sum(top[i] == lab for i in rep)
136
+ true = sum(gold[i] == lab for i in rep)
137
+ p = tp / pred if pred else None
138
+ r = tp / true if true else None
139
+ ok = [i for i in passed if top[i] == lab]
140
+ hits = sum(gold[i] == lab for i in ok)
141
+ wk, wa = sum(w[i] for i in passed), sum(w.values())
142
+ return {"n": true, "precision": _round4(p), "recall": _round4(r),
143
+ "f1": _round4(2 * p * r / (p + r)) if p and r else (0.0 if pred or true else None),
144
+ "tau": tau, "coverage": _round4(len(ok) / pred) if pred else None,
145
+ "precision_at_tau": _round4(hits / len(ok)) if ok else None,
146
+ "precision_lb": _round4(wilson_lb(hits, len(ok))),
147
+ "share_error_pp": _round4(abs(sum(w[i] for i in ok) / wk
148
+ - sum(w[i] for i in rep if gold[i] == lab) / wa)
149
+ * 100) if wk else None}
150
+
151
+
152
+ def _per_label(preds: Sequence[str | None], golds: list[str],
153
+ labels: list[str]) -> dict[str, dict[str, Any]]:
154
+ out = {}
155
+ for lab in labels:
156
+ tp = sum(p == lab == g for p, g in zip(preds, golds))
157
+ pred, true = sum(p == lab for p in preds), sum(g == lab for g in golds)
158
+ out[lab] = {"n": true, "precision": _round4(tp / pred) if pred else None,
159
+ "recall": _round4(tp / true) if true else None,
160
+ "f1": _round4(2 * tp / (pred + true)) if pred or true else None}
161
+ return out
162
+
163
+
164
+ def _macro(per: dict[str, dict[str, Any]]) -> float | None:
165
+ f1s = [v["f1"] for v in per.values() if v["f1"] is not None]
166
+ return _round4(sum(f1s) / len(f1s)) if f1s else None
167
+
168
+
169
+ def _report(raws: list[Dist], dists: list[Dist], preds: Sequence[str | None], golds: list[str],
170
+ labels: list[str]) -> dict[str, Any]:
171
+ """The numbers of one report: at the tuned tau, without a threshold, and off the shelf (raw
172
+ scores at the default threshold of 0.5)."""
173
+ tops = [top_label(d) for d in dists]
174
+ plain, tuned = _per_label(tops, golds, labels), _per_label(preds, golds, labels)
175
+ half = dict.fromkeys(labels, 0.5)
176
+ default = _per_label([_decide(d, half) for d in raws], golds, labels)
177
+ rows = list(zip(dists, golds))
178
+ return {"n": len(golds), "macro_f1": _macro(plain), "macro_f1_tuned": _macro(tuned),
179
+ "macro_f1_default": _macro(default), "kappa": _round4(kappa(list(zip(tops, golds)))),
180
+ "brier": _round4(brier(rows)), "nll": _round4(nll(rows)),
181
+ "coverage": _round4(sum(p is not None for p in preds) / len(preds)),
182
+ "per_label": {lab: {**plain[lab], "precision_at_tau": tuned[lab]["precision"],
183
+ "recall_at_tau": tuned[lab]["recall"],
184
+ "f1_at_tau": tuned[lab]["f1"],
185
+ "precision_default": default[lab]["precision"],
186
+ "recall_default": default[lab]["recall"]} for lab in labels}}
187
+
188
+
189
+ def _score(backend: Backend, reqs: Sequence[Request],
190
+ prev: dict[str, Any] | None = None) -> tuple[dict[str, Dist], dict[str, Any]]:
191
+ """(unit_id -> raw distribution, speed): saved scores, and the model only for units without
192
+ one. model_rev must be final by now, so run its prepare() first. The first call is the warm-up
193
+ (load plus one decision), timed apart; when this run timed no more calls than `prev` did,
194
+ prev's speed stands. Only a score of the same complete request can be reused."""
195
+ p = scores_path(reqs[0].task, backend.model_id, backend.model_rev, reqs[0].prompt_ver,
196
+ getattr(backend, "protocol", ""))
197
+ cached = {(r["unit_id"], r.get("input_hash")): r["dist"] for r in read_jsonl(p)}
198
+ saved: dict[str, Dist] = {}
199
+ times = []
200
+ for r in reqs:
201
+ digest = hashlib.sha256(json.dumps(asdict(r), sort_keys=True,
202
+ allow_nan=False).encode()).hexdigest()
203
+ if (r.unit_id, digest) in cached:
204
+ saved[r.unit_id] = cached[r.unit_id, digest]
205
+ else:
206
+ t0 = time.monotonic()
207
+ saved[r.unit_id] = backend.distribution(r)
208
+ times.append(time.monotonic() - t0)
209
+ append_jsonl(p, {"unit_id": r.unit_id, "input_hash": digest, "dist": saved[r.unit_id]})
210
+ warm = times[1:]
211
+ if prev and prev.get("n_timed", 0) >= len(warm):
212
+ return saved, {k: prev.get(k) for k in ("s_per_decision", "load_s", "peak_mb", "n_timed")}
213
+ peak = getattr(backend, "peak_memory_mb", None)
214
+ return saved, {"s_per_decision": _round4(sum(warm) / len(warm)) if warm else None,
215
+ "load_s": _round4(times[0]) if times else None, "n_timed": len(warm),
216
+ "peak_mb": peak() if peak and times else None}
217
+
218
+
219
+ def _fit_calibration(kind: str, raws: list[Dist], golds: list[str], yes: str) -> dict[str, Any]:
220
+ """Keyword arguments of core.calibrate for one way to calibrate, fitted on these units."""
221
+ if kind == "temperature":
222
+ return {"temperature": fit_temperature(raws, golds), "platt": None}
223
+ return {"temperature": 1.0, "platt": fit_platt(raws, golds, yes) if kind == "platt" else None}
224
+
225
+
226
+ def evaluate(task: str, backend: Backend, items: Sequence[_Item], precision: float, recall: float,
227
+ today: date, prev: dict[str, Any] | None = None) -> dict[str, Any] | None:
228
+ """Items never come from a final test day. The eval record of one backend (numbers), out of
229
+ fold; None when nothing is left to report on."""
230
+ labels = [c for c, _d in TASKS[task].candidates]
231
+ # the encoder's training days never calibrate it
232
+ items = [it for it in items
233
+ if not (isinstance(backend, Encoder) and in_encoder_train_split(it.req))]
234
+ if not items:
235
+ return None
236
+ reqs = [it.req for it in items]
237
+ golds = [it.gold for it in items]
238
+ w = [it.weight for it in items]
239
+ saved, speed = _score(backend, reqs, prev)
240
+ raws = [{lab: saved[r.unit_id].get(lab, 0.0) for lab in labels} for r in reqs]
241
+ folds = [fold(r) for r in reqs]
242
+ every = range(len(reqs))
243
+ split = [([i for i in every if folds[i] != k], [i for i in every if folds[i] == k])
244
+ for k in range(FOLDS)]
245
+
246
+ def fit(kind: str, on: Iterable[int]) -> dict[str, Any]:
247
+ on = list(on)
248
+ return _fit_calibration(kind, [raws[i] for i in on], [golds[i] for i in on], labels[0])
249
+
250
+ def out_of_fold(kind: str) -> list[Dist]:
251
+ out: list[Dist] = [{}] * len(reqs)
252
+ for train, test in split:
253
+ c = fit(kind, train)
254
+ for i in test:
255
+ out[i] = calibrate(raws[i], **c)
256
+ return out
257
+
258
+ # T = 1 unless a fit lowers the out-of-fold Brier. shortcut: the way to calibrate is picked on
259
+ # the folds it reports on (one pick of at most three), upgrade by nesting it if gold grows.
260
+ kinds = ("none", "temperature") + (("platt",) if TASKS[task].decision == "boolean" else ())
261
+ briers = {k: _round4(brier(list(zip(out_of_fold(k), golds)))) or 0.0 for k in kinds}
262
+ kind = min(kinds, key=briers.__getitem__)
263
+ dists = out_of_fold(kind)
264
+ preds: list[str | None] = [None] * len(reqs)
265
+ no_threshold: dict[str, list[str]] = {}
266
+ for k, (train, test) in enumerate(split):
267
+ c = fit(kind, train)
268
+ tau = fit_tau([(top_label(d), d[top_label(d)], golds[i]) for i in train
269
+ for d in [calibrate(raws[i], **c)]], precision, recall=recall)
270
+ missing = sorted(lab for lab in set(golds) if tau.get(lab) is None)
271
+ if test and missing:
272
+ no_threshold[str(k)] = missing
273
+ for i in test:
274
+ preds[i] = _decide(dists[i], tau)
275
+ saved_cal = fit(kind, every)
276
+ full = [calibrate(r, **saved_cal) for r in raws]
277
+ tau = fit_tau([(top_label(d), d[top_label(d)], g) for d, g in zip(full, golds)], precision,
278
+ recall=recall)
279
+ rep = _report(raws, dists, preds, golds, labels)
280
+ tops = [top_label(d) for d in dists]
281
+ passed = [i for i in every if preds[i]]
282
+ for lab, v in rep["per_label"].items():
283
+ v.update(_label_metrics(lab, list(every), tops, dict(enumerate(golds)), passed,
284
+ dict(enumerate(w)), tau.get(lab)))
285
+ ver = TASKS[task].ver
286
+ return {"task": task, "task_ver": ver, "backend": backend.backend,
287
+ "model_id": backend.model_id, "model_rev": backend.model_rev,
288
+ "size_mb": backend.size_mb, "prompt_ver": reqs[0].prompt_ver,
289
+ "calib_ver": f"{task}@{ver}/{today.isoformat()}", "calibration": kind, **saved_cal,
290
+ "brier_by_calibration": briers, "n_cal": len(reqs), "n_held": len(reqs),
291
+ "folds": {str(k): len(test) for k, (_t, test) in enumerate(split)},
292
+ "no_threshold_folds": no_threshold, **rep, **speed,
293
+ "ece_raw": _round4(ece([(max(r.values()), top_label(r) == g)
294
+ for r, g in zip(raws, golds)])),
295
+ "ece_cal": _round4(ece([(max(d.values()), top_label(d) == g)
296
+ for d, g in zip(dists, golds)])),
297
+ "reliability_raw": reliability([(max(r.values()), top_label(r) == g)
298
+ for r, g in zip(raws, golds)]),
299
+ "reliability_cal": reliability([(max(d.values()), top_label(d) == g)
300
+ for d, g in zip(dists, golds)])}
301
+
302
+
303
+ def _gold_units(cfg: Config, task: str, days: int, src: Source | None,
304
+ today: date | None = None) -> list[tuple[Request, str]]:
305
+ """(request, gold) for every gold unit still in the window, final test days included: each
306
+ caller takes its side of evalset.final_test."""
307
+ gold = gold_latest(task)
308
+ reqs = window_units(cfg, task, days, src, today)
309
+ found = [u for u in gold if u in reqs]
310
+ if len(found) < len(gold):
311
+ print(f"aut: label eval: {len(gold) - len(found)} gold units are outside the {days}-day "
312
+ "window; left out", file=sys.stderr)
313
+ return [(reqs[u], gold[u]) for u in found]
314
+
315
+
316
+ def _warn_thin(task: str, golds: list[str], what: str) -> None:
317
+ floor = FLOORS[task][0]
318
+ n = Counter(golds)
319
+ thin = {c: n[c] for c, _d in TASKS[task].candidates if n[c] < floor}
320
+ if thin:
321
+ print(f"aut: label eval: {what} is too small to mean much: under {floor} gold per label "
322
+ f"for {json.dumps(thin)}", file=sys.stderr)
323
+
324
+
325
+ def default_backends(cfg: Config, task: str) -> list[Backend]:
326
+ """Every model in the table (and label_model) that is fully on disk, plus the encoder once
327
+ trained. The others are printed with what it takes to fetch them."""
328
+ out: list[Backend] = []
329
+ for name in dict.fromkeys(list(MODELS) + [short_name(cfg.label_model)]):
330
+ b = make_backend(task, Choice(model_id=name), cfg.label_model, cfg.label_llama_server)
331
+ try:
332
+ getattr(b, "prepare", lambda: None)()
333
+ except BackendUnavailable as exc:
334
+ print(f"aut: label eval: {name} left out: {exc}", file=sys.stderr)
335
+ continue
336
+ out.append(b)
337
+ if encoder_path(task).exists():
338
+ out.append(Encoder(task))
339
+ return out
340
+
341
+
342
+ def _key(rec: dict[str, Any]) -> tuple[Any, ...]:
343
+ return rec.get("model_id"), rec.get("model_rev"), rec.get("prompt_ver")
344
+
345
+
346
+ def newest(task: str) -> dict[tuple[Any, ...], dict[str, Any]]:
347
+ """The newest record per model, revision and prompt of the current task version, the most
348
+ recent one last. Records from before the five folds have no Brier and do not count."""
349
+ out: dict[tuple[Any, ...], dict[str, Any]] = {}
350
+ for r in read_jsonl(eval_path(task, "eval")):
351
+ if r.get("task_ver") == TASKS[task].ver and "brier" in r:
352
+ out.pop(_key(r), None)
353
+ out[_key(r)] = r
354
+ return out
355
+
356
+
357
+ def run_eval(cfg: Config, task: str, days: int = LABEL_DAYS, *, src: Source | None = None,
358
+ backends: list[Backend] | None = None, now: datetime | None = None) -> list[dict[str, Any]]:
359
+ now = now or datetime.now(timezone.utc)
360
+ today = now.astimezone().date()
361
+ pool = [(r, g) for r, g in _gold_units(cfg, task, days, src, today) if not final_test(r)]
362
+ if not pool:
363
+ raise AutError(f"label eval: no gold labels for {task} in the last {days} days outside "
364
+ "the final test days (label some with `aut label review`)")
365
+ _warn_thin(task, [g for _q, g in pool], "the tuning gold")
366
+ w = weights(task, [r.unit_id for r, _g in pool])
367
+ items = [_Item(r, g, w[r.unit_id]) for r, g in pool]
368
+ before, recs = newest(task), []
369
+ for b in backends if backends is not None else default_backends(cfg, task):
370
+ try:
371
+ getattr(b, "prepare", lambda: None)()
372
+ rec = evaluate(task, b, items, cfg.label_precision, cfg.label_recall, today,
373
+ before.get((b.model_id, b.model_rev, items[0].req.prompt_ver)))
374
+ except BackendUnavailable as exc: # one model that cannot run here: the others still run
375
+ print(f"aut: label eval: {b.model_id} left out: {exc}", file=sys.stderr)
376
+ continue
377
+ finally:
378
+ getattr(b, "close", lambda: None)()
379
+ if rec is None:
380
+ print(f"aut: label eval: {b.model_id} has no gold outside its training days; left out",
381
+ file=sys.stderr)
382
+ continue
383
+ append_jsonl(eval_path(task, "eval"), {"ts": now.isoformat(), **rec})
384
+ if cfg.pg_dsn:
385
+ send_activity_rows(cfg.pg_dsn, [ActivityEvent(
386
+ event_id=make_event_id("eval", json.dumps(rec, sort_keys=True)),
387
+ ts=now.isoformat(), agent="label", kind="eval", session_id=None, turn_id=None,
388
+ attrs=rec, machine=cfg.machine)])
389
+ recs.append(rec)
390
+ return recs
391
+
392
+
393
+ def run_final(cfg: Config, task: str, model: str, days: int = LABEL_DAYS, *,
394
+ src: Source | None = None, backend: Backend | None = None,
395
+ now: datetime | None = None) -> dict[str, Any]:
396
+ """Score one model's saved calibration and tau on the final test days. Nothing fits here, and
397
+ only this function reads those days; the result goes to its own file, which nothing reads."""
398
+ now = now or datetime.now(timezone.utc)
399
+ rec = _saved_result(task, model)
400
+ pairs = [(r, g) for r, g in _gold_units(cfg, task, days, src, now.astimezone().date())
401
+ if final_test(r)]
402
+ if not pairs:
403
+ raise AutError(f"label eval --final: no gold labels for {task} on the final test days "
404
+ "yet (label more days with `aut label review`)")
405
+ reqs, golds = [r for r, _g in pairs], [g for _r, g in pairs]
406
+ if rec["prompt_ver"] != reqs[0].prompt_ver:
407
+ raise AutError(f"label eval --final: {model} was tuned on prompt {rec['prompt_ver']}; "
408
+ "run `aut label eval` again first")
409
+ _warn_thin(task, golds, "the final test set")
410
+ out = eval_path(task, "final")
411
+ looks = sum(r.get("model_id") == rec["model_id"] for r in read_jsonl(out))
412
+ if looks:
413
+ print(f"aut: label eval: {rec['model_id']} has seen the final test set {looks} times; "
414
+ "if you changed anything because of it, it no longer tests that", file=sys.stderr)
415
+ b = backend or make_backend(task, Choice(backend=rec["backend"], model_id=rec["model_id"],
416
+ model_rev=rec["model_rev"]), cfg.label_model,
417
+ cfg.label_llama_server)
418
+ try:
419
+ getattr(b, "prepare", lambda: None)()
420
+ saved, _speed = _score(b, reqs)
421
+ finally:
422
+ getattr(b, "close", lambda: None)()
423
+ labels = [c for c, _d in TASKS[task].candidates]
424
+ raws = [{lab: saved[r.unit_id].get(lab, 0.0) for lab in labels} for r in reqs]
425
+ tau = {k: v["tau"] for k, v in rec["per_label"].items()}
426
+ dists = [calibrate(r, rec["temperature"], rec.get("platt")) for r in raws]
427
+ res = {"ts": now.isoformat(), "task": task, "task_ver": rec["task_ver"],
428
+ "backend": rec["backend"], "model_id": rec["model_id"], "model_rev": rec["model_rev"],
429
+ "prompt_ver": rec["prompt_ver"], "calib_ver": rec["calib_ver"],
430
+ **_report(raws, dists, [_decide(d, tau) for d in dists], golds, labels)}
431
+ append_jsonl(out, res)
432
+ res["fails"] = failures(res, cfg.label_precision, cfg.label_recall)
433
+ return res
434
+
435
+
436
+ def final_line(r: dict[str, Any]) -> str:
437
+ fails = "; ".join(r["fails"]) or "none"
438
+ return (f"label eval --final {r['task']} {r['model_id']}: n {r['n']}, macro-F1 tuned "
439
+ f"{r['macro_f1_tuned']} (default 0.5: {r['macro_f1_default']}), Brier {r['brier']}, "
440
+ f"NLL {r['nll']}, coverage {r['coverage']}, floors missed: {fails}")
441
+
442
+
443
+ def failures(rec: dict[str, Any], precision: float, recall: float,
444
+ coverage: float | None = None) -> list[str]:
445
+ """Why a report misses the bar: a label under the precision or recall floor at its tau (a
446
+ label with no gold has no recall to judge), and, when coverage is given, the rest of the bar
447
+ (CONTRACT.md, the bake-off): coverage at tau, and no label's share on known items off by over
448
+ SHARE_ERROR_PP."""
449
+ out = []
450
+ for lab, v in rec["per_label"].items():
451
+ if v["precision_at_tau"] is not None and v["precision_at_tau"] < precision:
452
+ out.append(f"{lab} precision {v['precision_at_tau']}")
453
+ if v["recall_at_tau"] is not None and v["recall_at_tau"] < recall:
454
+ out.append(f"{lab} recall {v['recall_at_tau']}")
455
+ if coverage is not None and (v["share_error_pp"] or 0) > SHARE_ERROR_PP:
456
+ out.append(f"{lab} share error {v['share_error_pp']}")
457
+ if coverage is not None and (rec["coverage"] or 0) < coverage:
458
+ out.append(f"coverage {rec['coverage']}")
459
+ return out
460
+
461
+
462
+ def _fails(rec: dict[str, Any], cfg: Config) -> list[str]:
463
+ return failures(rec, cfg.label_precision, cfg.label_recall, cfg.label_coverage)
464
+
465
+
466
+ def pareto(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
467
+ """The rows no other row beats on quality (tuned macro-F1), speed and memory. An axis a row
468
+ does not know (null) is not compared."""
469
+ def axes(r: dict[str, Any]) -> tuple[Any, ...]:
470
+ return -(r.get("macro_f1_tuned") or 0), r.get("s_per_decision"), r.get("peak_mb")
471
+
472
+ def beats(a: dict[str, Any], b: dict[str, Any]) -> bool:
473
+ pairs = [(x, y) for x, y in zip(axes(a), axes(b)) if x is not None and y is not None]
474
+ return all(x <= y for x, y in pairs) and any(x < y for x, y in pairs)
475
+ return [r for r in rows if not any(beats(o, r) for o in rows)]
476
+
477
+
478
+ def recommend(rows: list[dict[str, Any]], cfg: Config) -> dict[str, Any] | None:
479
+ """Of the frontier of models that pass: the best quality (to two decimals: closer is noise on
480
+ gold this size), then the fastest."""
481
+ front = pareto([r for r in rows if not _fails(r, cfg)])
482
+ return min(front, key=lambda r: (-round(r.get("macro_f1_tuned") or 0, 2),
483
+ r.get("s_per_decision") or math.inf,
484
+ r.get("peak_mb") or math.inf), default=None)
485
+
486
+
487
+ def table(cfg: Config, task: str) -> list[str]:
488
+ """Quality, speed and memory per model. `*` on the Pareto frontier of the models that pass,
489
+ `!` fails a floor or the bar. Numbers are out of fold; none comes from the final test days."""
490
+ rows = sorted(newest(task).values(), key=lambda r: -(r.get("macro_f1_tuned") or 0))
491
+ front = pareto([r for r in rows if not _fails(r, cfg)])
492
+ lines = [f"label eval {task}: floors precision {cfg.label_precision}, recall "
493
+ f"{cfg.label_recall}, coverage {cfg.label_coverage}",
494
+ f" {'model':<46} {'rev':<8} {'F1':>6} {'F1@.5':>6} {'Brier':>6} {'NLL':>6} "
495
+ f"{'calib':<11} {'s/dec':>7} {'load s':>7} {'peak MB':>7} {'n':>5}"]
496
+ for r in rows:
497
+ mark = ("*" if any(r is o for o in front) else " ") + ("!" if _fails(r, cfg) else " ")
498
+ lines.append(f"{mark} {short_name(r['model_id']):<46} {r['model_rev'][:8]:<8} "
499
+ f"{r['macro_f1_tuned']!s:>6} {r['macro_f1_default']!s:>6} {r['brier']!s:>6} "
500
+ f"{r['nll']!s:>6} {r['calibration']:<11} {r.get('s_per_decision')!s:>7} "
501
+ f"{r.get('load_s')!s:>7} {r.get('peak_mb')!s:>7} {r['n']:>5}")
502
+ return lines + ["* Pareto frontier of the models that pass ! fails: "
503
+ + "; ".join(f"{short_name(r['model_id'])}: {', '.join(_fails(r, cfg))}"
504
+ for r in rows if _fails(r, cfg))]
505
+
506
+
507
+ def details(recs: list[dict[str, Any]]) -> list[str]:
508
+ """Per label and reliability of the models just evaluated."""
509
+ def rel(rows: list[dict[str, Any]]) -> str:
510
+ return " ".join(f"{g['p']}:{g['right']}" for g in rows)
511
+
512
+ out = []
513
+ for r in recs:
514
+ out += ["", f"{r['model_id']}: {r['calibration']} (T {r['temperature']}, platt "
515
+ f"{json.dumps(r['platt'])}), Brier {r['brier_by_calibration']}, folds "
516
+ f"{r['folds']}, no threshold in folds {r['no_threshold_folds'] or 'none'}",
517
+ f" {'label':<16} {'n':>4} {'tau':>6} {'P':>6} {'R':>6} "
518
+ f"at 0.5: {'P':>6} {'R':>6}"]
519
+ out += [f" {lab:<16} {v['n']:>4} {_round4(v['tau'])!s:>6} {v['precision_at_tau']!s:>6} "
520
+ f"{v['recall_at_tau']!s:>6} {v['precision_default']!s:>6} "
521
+ f"{v['recall_default']!s:>6}" for lab, v in r["per_label"].items()]
522
+ out += [f" reliability raw (mean p:share right) {rel(r['reliability_raw'])}",
523
+ f" reliability calibrated {rel(r['reliability_cal'])}"]
524
+ return out
525
+
526
+
527
+ def _saved_result(task: str, model: str) -> dict[str, Any]:
528
+ """The newest record of a model id (or a short name from backends.MODELS)."""
529
+ known = find_model(model)
530
+ repo = known.repo if known else model
531
+ match = [r for r in newest(task).values() if r["model_id"] == repo]
532
+ if not match:
533
+ raise AutError(f"label: no eval record of {model} for {task}; run `aut label eval` "
534
+ f"first. Known: {sorted({r['model_id'] for r in newest(task).values()})}")
535
+ return match[-1]
536
+
537
+
538
+ def choose(cfg: Config, task: str, model: str | None = None) -> dict[str, Any] | None:
539
+ """With a model: write its saved result into label.json; other tasks untouched. Without one:
540
+ print the table and a recommendation, and write nothing."""
541
+ if model is None:
542
+ rows = list(newest(task).values())
543
+ if not rows:
544
+ raise AutError(f"label choose: no eval record for {task}; run `aut label eval` first")
545
+ rec = recommend(rows, cfg)
546
+ print("\n".join(table(cfg, task)))
547
+ print(f"recommended: aut label choose --task {task} --model "
548
+ f"{short_name(rec['model_id'])}" if rec else
549
+ "recommended: none, no model passes the floors. They stay as they are: label more "
550
+ "gold, or try other models.")
551
+ return None
552
+ win = _saved_result(task, model)
553
+ fails = _fails(win, cfg)
554
+ if fails:
555
+ print(f"aut: warning: {win['model_id']} fails the floors for {task}: {', '.join(fails)}. "
556
+ "Labels without a tau stay unknown.", file=sys.stderr)
557
+ p = choice_path()
558
+ try:
559
+ obj = read_choices()
560
+ except (OSError, ValueError) as exc: # overwriting would lose every other task's choice
561
+ raise AutError(f"label choose: {p} is unusable ({exc}); nothing written. Fix or move "
562
+ "it aside and run choose again for each task.") from None
563
+ obj.setdefault("tasks", {})[task] = {
564
+ "backend": win["backend"], "model_id": win["model_id"], "model_rev": win["model_rev"],
565
+ "temperature": win["temperature"], "platt": win.get("platt"),
566
+ "calib_ver": win["calib_ver"], "prompt_ver": win["prompt_ver"],
567
+ "tau": {k: v["tau"] for k, v in win["per_label"].items() if v["tau"] is not None}}
568
+ atomic_write_text(p, json.dumps(obj, indent=2, sort_keys=True))
569
+ return obj["tasks"][task]
570
+
571
+
572
+ MIN_UPDATES = 2000
573
+
574
+
575
+ def _fit(data: list[tuple[list[int], list[float], float]], k: int, epochs: int = 12,
576
+ lr: float = 0.3, l2: float = 1e-4, seed: int = 0) -> dict[int, list[float]]:
577
+ """Multinomial logistic regression by SGD on (features, target distribution, weight)."""
578
+ w: dict[int, list[float]] = {}
579
+ rng, data = random.Random(seed), list(data) # nosec B311 # SGD shuffle order, not security
580
+ # a small gold set gets more passes: 12 passes over 20 items underfit even a separable set
581
+ epochs = max(epochs, -(-MIN_UPDATES // max(1, len(data))))
582
+ for ep in range(epochs):
583
+ rng.shuffle(data)
584
+ step = lr / (1 + ep)
585
+ for feats, target, wt in data:
586
+ rows = [w.setdefault(f, [0.0] * k) for f in feats]
587
+ p = softmax([sum(r[j] for r in rows) for j in range(k)])
588
+ g = [(p[j] - target[j]) * wt * step for j in range(k)]
589
+ for r in rows:
590
+ for j in range(k):
591
+ r[j] -= g[j] + step * l2 * r[j]
592
+ return w
593
+
594
+
595
+ def _training_data(gold: dict[str, str], silver: dict[str, dict[str, Any]], train: dict[str, Request],
596
+ labels: list[str]) -> list[tuple[list[int], list[float], float]]:
597
+ """(features, target distribution, weight) of each gold and silver unit of the training days."""
598
+ data = []
599
+ for u, g in gold.items():
600
+ if u in train:
601
+ data.append((features(train[u].facts), [float(x == g) for x in labels], GOLD_WEIGHT))
602
+ for u, ts in silver.items():
603
+ if u in train and u not in gold and ts:
604
+ avg = [sum(d.get(x, 0.0) for d in ts.values()) / len(ts) for x in labels]
605
+ data.append((features(train[u].facts), avg, 1.0))
606
+ return data
607
+
608
+
609
+ def _macro_f1(pairs: list[tuple[str, str]], labels: list[str]) -> float | None:
610
+ f1s = []
611
+ for lab in labels:
612
+ tp = sum(a == lab == b for a, b in pairs)
613
+ pred, true = sum(a == lab for a, _ in pairs), sum(b == lab for _, b in pairs)
614
+ if pred or true:
615
+ f1s.append(2 * tp / (pred + true))
616
+ return _round4(sum(f1s) / len(f1s)) if f1s else None
617
+
618
+
619
+ def train_encoder(cfg: Config, task: str, days: int = LABEL_DAYS, *, src: Source | None = None,
620
+ now: datetime | None = None) -> dict[str, Any]:
621
+ """Fit the encoder on the gold (GOLD_WEIGHT) and silver (teachers averaged) of its training
622
+ days (in_encoder_train_split), never on a final test day nor on a day eval calibrates it with;
623
+ report accuracy and macro-F1 on the gold of those other days."""
624
+ labels = [c for c, _d in TASKS[task].candidates]
625
+ today = (now or datetime.now(timezone.utc)).astimezone().date()
626
+ reqs = window_units(cfg, task, days, src, today)
627
+ gold, silver = gold_latest(task), silver_by_unit(task)
628
+ open_days = {u: r for u, r in reqs.items() if not final_test(r)}
629
+ train = {u: r for u, r in open_days.items() if in_encoder_train_split(r)}
630
+ test = {u for u in gold if u in open_days and u not in train}
631
+ data = _training_data(gold, silver, train, labels)
632
+ if not data:
633
+ raise AutError(f"label train-encoder: no gold or silver labels for {task} yet")
634
+ w = _fit(data, len(labels))
635
+ # atomic: a crash mid-write must not leave a corrupt encoder that the nightly run then loads
636
+ atomic_write_text(encoder_path(task), json.dumps(
637
+ {"task": task, "task_ver": TASKS[task].ver, "labels": labels,
638
+ "w": {str(f): [round(x, 5) for x in r] for f, r in w.items()}}))
639
+ enc = Encoder(task)
640
+ pairs = [(top_label(enc.distribution(reqs[u])), gold[u]) for u in sorted(test)]
641
+ return {"model_id": enc.model_id, "model_rev": enc.model_rev, "trained_on": len(data),
642
+ "held_out": len(pairs),
643
+ "accuracy": _round4(sum(a == b for a, b in pairs) / len(pairs)) if pairs else None,
644
+ "macro_f1": _macro_f1(pairs, labels)}
645
+
646
+
647
+ def print_eval(cfg: Config, task: str, records: list[dict[str, Any]]) -> None:
648
+ print("\n".join(table(cfg, task) + details(records)))
649
+
650
+
651
+ def print_final(r: dict[str, Any]) -> None:
652
+ print(final_line(r))
653
+
654
+
655
+ def print_choose(task: str, c: dict[str, Any]) -> None:
656
+ print(f"label choose {task}: {c['backend']} {c['model_id']} (T {c['temperature']}, platt "
657
+ f"{json.dumps(c['platt'])}, tau {json.dumps(c['tau'])})")
658
+
659
+
660
+ def print_train_encoder(task: str, r: dict[str, Any]) -> None:
661
+ print(f"label train-encoder {task}: {json.dumps(r)}")