agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,355 @@
1
+ """The decision interface every label task uses, its calibration, and the stored label row.
2
+
3
+ A backend returns a raw distribution over the request's candidates. Calibration (temperature
4
+ scaling, or Platt scaling on a yes/no task, fitted per task and model on gold labels, and only
5
+ when it helps) turns it into the calibrated distribution, and a per-label threshold tau decides
6
+ whether the top answer counts: below tau the row is still written,
7
+ with below_tau = true, and readers treat it as unknown. "Unknown" is never a candidate.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import math
14
+ import os
15
+ import sys
16
+ from dataclasses import dataclass, field
17
+ from datetime import date, datetime, timedelta, timezone
18
+ from pathlib import Path
19
+ from typing import Any, Callable, Iterable, Iterator, Protocol, runtime_checkable
20
+
21
+ from ..config import config_path
22
+ from ..event import ActivityEvent, make_event_id
23
+
24
+ PROMPT_VER = 2 # bump when render_prompt() or the chat prompt changes; task_ver covers candidates/definitions
25
+ LETTERS = "ABCDEFGHIJ"
26
+ TAU_FLOOR = 0.5
27
+
28
+
29
+ def local_midnight(d: date) -> datetime:
30
+ return datetime(d.year, d.month, d.day).astimezone()
31
+
32
+
33
+ def local_days(since: datetime, until: datetime) -> Iterator[date]:
34
+ """The local dates from `since`'s up to, not including, `until`'s."""
35
+ d, end = since.astimezone().date(), until.astimezone().date()
36
+ while d < end:
37
+ yield d
38
+ d += timedelta(days=1)
39
+
40
+
41
+ def data_dir() -> Path:
42
+ """Where label data that never leaves this machine lives (gold and silver under eval/)."""
43
+ base = os.environ.get("XDG_DATA_HOME", str(Path.home() / ".local" / "share"))
44
+ return Path(base) / "aut"
45
+
46
+
47
+ def choice_path() -> Path:
48
+ """The chosen backend, model, temperature and tau per task (written by `aut label choose`)."""
49
+ return config_path().parent / "label.json"
50
+
51
+
52
+ @dataclass(slots=True)
53
+ class Request:
54
+ task: str
55
+ task_ver: int
56
+ unit: str # steer | episode_day | aw_block | episode | aw_title
57
+ unit_id: str
58
+ ts: str # the unit's own time (ISO-8601): the label row's ts
59
+ facts: dict[str, Any] # redacted; only explicit private replay jobs retain the facts
60
+ question: str
61
+ decision: str # choice | boolean
62
+ candidates: list[dict[str, Any]] # [{"id", "desc"}]: the task's closed list
63
+ none_fits: str | None # the explicit out-of-list candidate ("other"), or None
64
+ rev: str | None = None # the unit's version, when a changed unit is re-asked (episode)
65
+
66
+ @property
67
+ def prompt_ver(self) -> str:
68
+ return f"{self.task}@{self.task_ver}/p{PROMPT_VER}"
69
+
70
+
71
+ @dataclass(slots=True)
72
+ class Response:
73
+ dist: dict[str, float] # calibrated, over every candidate
74
+ chosen: str
75
+ prob: float
76
+ below_tau: bool
77
+ model_id: str
78
+ model_rev: str
79
+ prompt_ver: str
80
+ calib_ver: str
81
+
82
+
83
+ @dataclass(slots=True)
84
+ class Choice:
85
+ """One task's winner. The default is uncalibrated: T = 1 and no tau, so nothing passes."""
86
+ backend: str | None = None # None: nothing recorded, the model table decides
87
+ model_id: str | None = None
88
+ model_rev: str | None = None
89
+ temperature: float = 1.0
90
+ platt: dict[str, Any] | None = None # {"yes", "a", "b"}: replaces the temperature
91
+ # label -> tau; a missing or null label never passes
92
+ tau: dict[str, float | None] = field(default_factory=dict)
93
+ calib_ver: str = "none"
94
+
95
+
96
+ class Backend(Protocol):
97
+ """What a decision reads. model_rev and size_mb may only be final after the first distribution
98
+ call, and model_rev goes into the label row's event_id: read them after scoring. A backend may
99
+ add prepare() (find the weights and fix model_rev without loading; the bake-off calls it first,
100
+ because saved scores are keyed on model_rev), peak_memory_mb(), close(), and a `protocol`
101
+ string for the saved-scores key."""
102
+ backend: str # what `aut label choose` writes, what make_backend reads
103
+ model_id: str
104
+ model_rev: str
105
+ size_mb: float | None
106
+
107
+ def distribution(self, req: Request) -> dict[str, float]: ...
108
+
109
+
110
+ @runtime_checkable
111
+ class NoteBackend(Backend, Protocol):
112
+ """A backend that can also write free text, for the private nightly note."""
113
+
114
+ def generate(self, prompt: str, max_tokens: int = 300) -> str: ...
115
+
116
+
117
+ def read_choices() -> dict[str, Any]:
118
+ """label.json; {} when there is none yet. Raises OSError or ValueError when it is unusable."""
119
+ try:
120
+ obj = json.loads(choice_path().read_text())
121
+ except FileNotFoundError:
122
+ return {}
123
+ if not isinstance(obj, dict) or not isinstance(obj.get("tasks", {}), dict):
124
+ raise ValueError("unexpected shape")
125
+ return obj
126
+
127
+
128
+ def _is_number(x: object) -> bool:
129
+ return isinstance(x, (int, float)) and not isinstance(x, bool)
130
+
131
+
132
+ def _check_entry(task: str, c: object) -> None:
133
+ """Raises ValueError for a label.json entry whose values decide() could not use."""
134
+ if not isinstance(c, dict):
135
+ raise ValueError(f"its entry for {task} is not a table")
136
+ t, tau = c.get("temperature", 1.0), c.get("tau", {})
137
+ if not _is_number(t) or t <= 0:
138
+ raise ValueError(f"the temperature of {task} is not a positive number")
139
+ if not isinstance(tau, dict) or not all(v is None or _is_number(v) for v in tau.values()):
140
+ raise ValueError(f"the tau of {task} is not a table of numbers")
141
+ platt = c.get("platt")
142
+ if platt is not None and not (isinstance(platt, dict) and isinstance(platt.get("yes"), str)
143
+ and _is_number(platt.get("a")) and _is_number(platt.get("b"))):
144
+ raise ValueError(f"the platt of {task} is not a table of yes, a and b")
145
+
146
+
147
+ def load_choice(task: str, task_ver: int | None = None) -> Choice:
148
+ try:
149
+ c = read_choices().get("tasks", {}).get(task)
150
+ if c is not None:
151
+ _check_entry(task, c)
152
+ if task_ver is not None and c.get("prompt_ver") != f"{task}@{task_ver}/p{PROMPT_VER}":
153
+ raise ValueError(f"the calibration of {task} belongs to another prompt version")
154
+ except (OSError, ValueError) as exc:
155
+ print(f"aut: warning: {choice_path()} is unusable ({exc}); {task} runs uncalibrated, so "
156
+ "none of its labels pass tau. Fix it or move it aside and re-run `aut label choose`.", file=sys.stderr)
157
+ return Choice()
158
+ if c is None:
159
+ return Choice()
160
+ return Choice(**{k: c[k] for k in Choice.__slots__ if k in c})
161
+
162
+
163
+ def render_prompt(req: Request, order: list[str]) -> str:
164
+ """The prompt for one candidate order: candidates as letters, the facts, a one-letter answer."""
165
+ desc = {c["id"]: c["desc"] for c in req.candidates}
166
+ lines = [req.question, "", "Facts:"]
167
+ lines += [f"- {k}: {v if isinstance(v, str) else json.dumps(v, ensure_ascii=False)}"
168
+ for k, v in req.facts.items()]
169
+ lines += ["", "Options:"] + [f"{LETTERS[i]}. {c}: {desc[c]}" for i, c in enumerate(order)]
170
+ lines += ["", f"Answer with one letter ({', '.join(LETTERS[:len(order)])}) and nothing else."]
171
+ return "\n".join(lines)
172
+
173
+
174
+ def softmax(xs: list[float]) -> list[float]:
175
+ m = max(xs)
176
+ e = [math.exp(x - m) for x in xs]
177
+ s = sum(e)
178
+ return [v / s for v in e]
179
+
180
+
181
+ def top_label(dist: dict[str, float]) -> str:
182
+ return max(dist, key=lambda label: dist[label])
183
+
184
+
185
+ def letter_distribution(letter_logits: Callable[[str, list[str]], list[float]],
186
+ req: Request) -> dict[str, float]:
187
+ """letter_logits(prompt, letters) -> one logit per letter at the answer position. Scores the
188
+ original and the reversed candidate order and averages the two (letter position bias)."""
189
+ ids = [c["id"] for c in req.candidates]
190
+ if not 1 <= len(ids) <= len(LETTERS):
191
+ raise ValueError(f"{req.task}: {len(ids)} candidates, at most {len(LETTERS)}")
192
+ out = dict.fromkeys(ids, 0.0)
193
+ orders = (ids, ids[::-1])
194
+ for order in orders:
195
+ for c, p in zip(order, softmax(letter_logits(render_prompt(req, order),
196
+ list(LETTERS[:len(order)])))):
197
+ out[c] += p / len(orders)
198
+ return out
199
+
200
+
201
+ def temper(dist: dict[str, float], t: float) -> dict[str, float]:
202
+ """Temperature scaling of a distribution: softmax(log p / t)."""
203
+ keys = list(dist)
204
+ return dict(zip(keys, softmax([math.log(max(dist[k], 1e-12)) / t for k in keys])))
205
+
206
+
207
+ def fit_temperature(dists: list[dict[str, Any]], golds: list[str], lo: float = 0.05,
208
+ hi: float = 20.0) -> float:
209
+ """The T that minimises the gold labels' negative log likelihood (golden section on log T;
210
+ the NLL of temperature scaling is unimodal in it). 1.0 without data."""
211
+ if not dists:
212
+ return 1.0
213
+
214
+ def nll(lt: float) -> float:
215
+ t = math.exp(lt)
216
+ return -sum(math.log(max(temper(d, t)[g], 1e-12)) for d, g in zip(dists, golds))
217
+
218
+ a, b, r = math.log(lo), math.log(hi), (math.sqrt(5) - 1) / 2
219
+ for _ in range(80):
220
+ c, d = b - r * (b - a), a + r * (b - a)
221
+ if nll(c) < nll(d):
222
+ b = d
223
+ else:
224
+ a = c
225
+ return round(math.exp((a + b) / 2), 4)
226
+
227
+
228
+ def _sigmoid(x: float) -> float:
229
+ return 1 / (1 + math.exp(-max(min(x, 35.0), -35.0)))
230
+
231
+
232
+ def _log_odds(p: float) -> float:
233
+ p = min(max(p, 1e-6), 1 - 1e-6)
234
+ return math.log(p / (1 - p))
235
+
236
+
237
+ def platt_scale(dist: dict[str, float], platt: dict[str, Any]) -> dict[str, float]:
238
+ """Platt scaling of a yes/no distribution: sigmoid(a * log-odds of "yes" + b)."""
239
+ q = _sigmoid(platt["a"] * _log_odds(dist[platt["yes"]]) + platt["b"])
240
+ return {k: q if k == platt["yes"] else 1 - q for k in dist}
241
+
242
+
243
+ def calibrate(raw: dict[str, float], temperature: float = 1.0,
244
+ platt: dict[str, Any] | None = None) -> dict[str, float]:
245
+ return platt_scale(raw, platt) if platt else temper(raw, temperature)
246
+
247
+
248
+ def fit_platt(dists: list[dict[str, float]], golds: list[str], yes: str,
249
+ l2: float = 1e-3) -> dict[str, Any]:
250
+ """The a and b that minimise the gold labels' NLL on the log-odds of `yes` (damped Newton; a
251
+ small pull towards a = 1, b = 0 keeps separable data finite). The identity without data."""
252
+ xy = [(_log_odds(d[yes]), float(g == yes)) for d, g in zip(dists, golds)]
253
+
254
+ def loss(a: float, b: float) -> float:
255
+ return l2 / 2 * ((a - 1) ** 2 + b ** 2) - sum(
256
+ math.log(max(q if y else 1 - q, 1e-12))
257
+ for x, y in xy for q in [_sigmoid(a * x + b)])
258
+
259
+ a, b = 1.0, 0.0
260
+ for _ in range(50 if xy else 0):
261
+ ga, gb, haa, hab, hbb = l2 * (a - 1), l2 * b, l2, 0.0, l2
262
+ for x, y in xy:
263
+ q = _sigmoid(a * x + b)
264
+ w = q * (1 - q)
265
+ ga, gb = ga + (q - y) * x, gb + q - y
266
+ haa, hab, hbb = haa + w * x * x, hab + w * x, hbb + w
267
+ det = haa * hbb - hab * hab
268
+ da, db = (hbb * ga - hab * gb) / det, (haa * gb - hab * ga) / det
269
+ step, base = 1.0, loss(a, b)
270
+ while step > 1e-4 and loss(a - step * da, b - step * db) > base:
271
+ step /= 2
272
+ a, b = a - step * da, b - step * db
273
+ if abs(step * da) + abs(step * db) < 1e-6:
274
+ break
275
+ return {"yes": yes, "a": round(a, 4), "b": round(b, 4)}
276
+
277
+
278
+ def fit_tau(items: list[tuple[str, float, str]], precision: float = 0.90,
279
+ floor: float = TAU_FLOOR, recall: float = 0.0) -> dict[str, float | None]:
280
+ """items: (chosen, p_cal, gold). Per chosen label: of the cuts between distinct p, no lower than
281
+ `floor`, the one with the best F1 among those that reach `precision` and `recall` on the items
282
+ chosen as that label with p >= it, the lowest on a tie. None when no cut does: the floors are
283
+ never relaxed, and the label never passes."""
284
+ by: dict[str, list[tuple[float, bool]]] = {}
285
+ true: dict[str, int] = {}
286
+ for chosen, p, gold in items:
287
+ by.setdefault(chosen, []).append((p, chosen == gold))
288
+ true[gold] = true.get(gold, 0) + 1
289
+ out: dict[str, float | None] = {}
290
+ for label, rows in by.items():
291
+ rows.sort(key=lambda r: -r[0])
292
+ n_true = true.get(label, 0)
293
+ best, top, ok = None, -1.0, 0
294
+ for i, (p, hit) in enumerate(rows):
295
+ if p < floor:
296
+ break
297
+ ok += hit
298
+ below = rows[i + 1][0] if i + 1 < len(rows) else None
299
+ f1 = 2 * ok / (i + 1 + n_true)
300
+ if (below is None or below < p) and ok / (i + 1) >= precision \
301
+ and ok / max(n_true, 1) >= recall and f1 >= top:
302
+ best, top = (floor if below is not None and below < floor else p), f1
303
+ out[label] = best
304
+ return out
305
+
306
+
307
+ def decide(req: Request, raw: dict[str, float], choice: Choice, model_id: str,
308
+ model_rev: str) -> Response:
309
+ dist = calibrate(raw, choice.temperature, choice.platt)
310
+ chosen = max(dist, key=dist.__getitem__)
311
+ tau = choice.tau.get(chosen)
312
+ return Response(dist=dist, chosen=chosen, prob=dist[chosen],
313
+ below_tau=tau is None or dist[chosen] < tau, model_id=model_id,
314
+ model_rev=model_rev, prompt_ver=req.prompt_ver, calib_ver=choice.calib_ver)
315
+
316
+
317
+ def label_event(req: Request, res: Response, labelled_at: str | None = None) -> ActivityEvent:
318
+ """The activity_event row of one decision: enums, numbers and versions, never the facts.
319
+ Same unit, task version, model and calibration -> same id, so a rerun is a no-op at the sink;
320
+ a new model, prompt or calibration (or a changed episode, req.rev) adds a row and never
321
+ overwrites one. labelled_at (UTC, fixed width) is the run's start: agents.label_latest takes
322
+ the newest one, so a later calibrated row wins over an earlier uncalibrated one."""
323
+ labelled_at = labelled_at or datetime.now(timezone.utc).isoformat(timespec="milliseconds")
324
+ return ActivityEvent(
325
+ event_id=make_event_id("label", req.task, req.task_ver, req.unit_id, res.model_id,
326
+ res.model_rev, res.prompt_ver, res.calib_ver,
327
+ *((req.rev,) if req.rev else ())),
328
+ ts=req.ts, agent="label", kind="label", session_id=None, turn_id=None,
329
+ attrs={"task": req.task, "task_ver": req.task_ver, "unit": req.unit,
330
+ "unit_id": req.unit_id, "label": res.chosen, "p": round(res.prob, 4),
331
+ "dist": {k: round(v, 4) for k, v in res.dist.items()},
332
+ "below_tau": res.below_tau, "model_id": res.model_id, "model_rev": res.model_rev,
333
+ "prompt_ver": res.prompt_ver, "calib_ver": res.calib_ver,
334
+ "labelled_at": labelled_at})
335
+
336
+
337
+ def current_labels(rows: Iterable[tuple[Any, ...]]) -> list[dict[str, Any]]:
338
+ """The one current label per (task, unit, unit_id) of (event_id, ts, attrs) rows, picked as
339
+ agents.label_latest picks it: newest task_ver, then newest labelled_at (a row without one
340
+ loses), then event_id. A row with no below_tau counts as below: readers treat it as unknown.
341
+ Working estimates have separate consumers and never enter calibrated selection."""
342
+ best: dict[tuple[Any, ...], tuple[Any, ...]] = {}
343
+ for eid, ts, a in rows:
344
+ a = a or {}
345
+ if str(a.get("task") or "").startswith("working."):
346
+ continue
347
+ key = (a.get("task"), a.get("unit"), a.get("unit_id"))
348
+ rank = (int(a.get("task_ver") or 0), a.get("labelled_at") or "", eid)
349
+ if key not in best or rank > best[key][0]:
350
+ below = a.get("below_tau")
351
+ best[key] = (rank, {"task": key[0], "unit": key[1], "unit_id": key[2], "ts": ts,
352
+ "label": a.get("label"), "p": a.get("p"),
353
+ "below_tau": True if below is None else bool(below),
354
+ "model_id": a.get("model_id"), "task_ver": rank[0]})
355
+ return [v for _r, v in best.values()]
@@ -0,0 +1,92 @@
1
+ """Opt-in local outcome catch-up with acknowledged, resumable private coverage progress."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import re
7
+ import time
8
+ from contextlib import nullcontext
9
+ from pathlib import Path
10
+ from typing import TYPE_CHECKING, Any, Callable
11
+
12
+ from ..config import AutError
13
+ from ..event import ActivityEvent
14
+ from ..state import LockBusy, _locked, atomic_write_text
15
+ from .backends import BackendUnavailable
16
+ from .core import Choice, Request, data_dir, decide, label_event
17
+ from .local import LocalBackend, decorate
18
+ from .outcome_coverage import coverage_key, pending_outcomes
19
+
20
+ if TYPE_CHECKING:
21
+ from .run import _Run
22
+
23
+
24
+ def _load(path: Path, machine: str) -> tuple[set[str], bool]:
25
+ try:
26
+ value = json.loads(path.read_text())
27
+ except FileNotFoundError:
28
+ return set(), False
29
+ except (OSError, ValueError) as exc:
30
+ raise AutError("label: outcome coverage progress is unreadable; it was not reset") from exc
31
+ if (not isinstance(value, dict) or value.get("version") != 1
32
+ or value.get("machine") != machine or type(value.get("next_mature")) is not bool
33
+ or not isinstance(value.get("acknowledged"), list)
34
+ or any(not isinstance(k, str) or not re.fullmatch(r"[a-f0-9]{64}", k)
35
+ for k in value["acknowledged"])):
36
+ raise AutError("label: invalid outcome coverage progress; it was not reset")
37
+ return set(value["acknowledged"]), value["next_mature"]
38
+
39
+
40
+ def covered_outcomes(run: _Run, requests: list[Request], backend: LocalBackend, choice: Choice,
41
+ report: dict[str, Any], batch_size: int,
42
+ deliver: Callable[[str, list[ActivityEvent]], Any]) -> None:
43
+ path = data_dir() / "label" / "outcome-coverage.json"
44
+ try:
45
+ with nullcontext() if run.dry_run else _locked(path, blocking=False):
46
+ done, next_mature = _load(path, run.cfg.machine)
47
+ pending = pending_outcomes(requests, done, backend.policy_id, start_mature=next_mature)
48
+ current = run.src.outcome_revisions([r for key, r in pending])
49
+ eligible = [(key, r) for key, r in pending if current.get(r.unit_id) == r.rev]
50
+ report["skipped_stale_outcome"] += len(pending) - len(eligible)
51
+ backend.prepare_requests([r for key, r in eligible], run.deadline)
52
+ report["provisional"] = 1
53
+ batch: list[tuple[str, Request, ActivityEvent]] = []
54
+
55
+ def flush() -> None:
56
+ nonlocal next_mature
57
+ revisions = run.src.outcome_revisions([r for key, r, ev in batch])
58
+ valid = [(key, r, ev) for key, r, ev in batch
59
+ if revisions.get(r.unit_id) == r.rev]
60
+ report["skipped_stale_outcome"] += len(batch) - len(valid)
61
+ if valid and not run.dry_run:
62
+ deliver(run.dsn, [ev for key, r, ev in valid])
63
+ # Unknown is an acknowledged estimate too. Never retry it as success hunting.
64
+ done.update(key for key, r, ev in valid)
65
+ next_mature = not coverage_key(valid[-1][1], backend.policy_id)[1]
66
+ atomic_write_text(path, json.dumps({"version": 1, "machine": run.cfg.machine,
67
+ "acknowledged": sorted(done), "next_mature": next_mature}, sort_keys=True))
68
+ for key, r, ev in valid:
69
+ report["rows"] += 1
70
+ report["below_tau"] += ev.attrs["below_tau"]
71
+ report["labels"][ev.attrs["label"]] += 1
72
+ batch.clear()
73
+
74
+ try:
75
+ for key, request in eligible:
76
+ if time.monotonic() >= run.deadline:
77
+ report["cut"] = 1
78
+ break
79
+ raw = backend.distribution(request)
80
+ result = decide(request, raw, choice, backend.model_id, backend.model_rev)
81
+ event = decorate(label_event(request, result, run.labelled_at), request,
82
+ backend, result.prob)
83
+ event.machine = run.cfg.machine
84
+ batch.append((key, request, event))
85
+ if len(batch) >= batch_size:
86
+ flush()
87
+ except BackendUnavailable:
88
+ flush() # Preserve completed estimates when a later model batch exhausts its budget.
89
+ raise
90
+ flush()
91
+ except LockBusy as exc:
92
+ raise BackendUnavailable("label: outcome coverage is already running") from exc
@@ -0,0 +1,154 @@
1
+ """The label files that never leave this machine, and the unit order the teacher and the gold page
2
+ share.
3
+
4
+ Under core.data_dir() / "eval": <task>.silver.jsonl (teacher distributions: unit_id, teacher,
5
+ dist, date, task_ver), <task>.jsonl (gold: unit_id, gold, labeller, date, task_ver),
6
+ <task>.eval.jsonl (one bake-off record per backend, numbers only), <task>.final.jsonl (results on
7
+ the final test days) and scores/ (one file per task, model, revision and prompt: unit_id and raw
8
+ dist). No row holds text: a unit's text is rebuilt from this machine's logs (tasks.units) whenever
9
+ it is needed.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import hashlib
15
+ import itertools
16
+ import json
17
+ import os
18
+ import random
19
+ import re
20
+ from collections import defaultdict
21
+ from datetime import date, datetime, timedelta
22
+ from pathlib import Path
23
+ from typing import Any, Callable, TypeVar
24
+
25
+ from ..config import LABEL_DAYS, AutError, Config
26
+ from ..state import atomic_write_text, parse_record
27
+ from .core import Request, data_dir, local_midnight, top_label
28
+ from .tasks import TASKS, Source, units
29
+
30
+ T = TypeVar("T")
31
+ # Gold per label, (floor, target), as the gold page shows them. AW titles: 150 per class.
32
+ FLOORS = {"work_category": (30, 50), "steer_intent": (30, 50), "soft_outcome": (30, 30),
33
+ "aw_title_relevance": (150, 150), "writing_activity": (30, 50)}
34
+ FOLDS = 5
35
+ FINAL_PCT = 20 # share of the days kept for `aut label eval --final`
36
+
37
+
38
+ def eval_path(task: str, kind: str) -> Path:
39
+ """kind: silver | gold | eval | final."""
40
+ return data_dir() / "eval" / (f"{task}.jsonl" if kind == "gold" else f"{task}.{kind}.jsonl")
41
+
42
+
43
+ def scores_path(task: str, model_id: str, model_rev: str, prompt_ver: str,
44
+ protocol: str = "") -> Path:
45
+ """The saved raw scores of one task, model, revision, prompt and decision protocol."""
46
+ parts = (task, model_id, model_rev, prompt_ver) + ((protocol,) if protocol else ())
47
+ key = hashlib.sha256("\0".join(parts).encode()).hexdigest()
48
+ name = re.sub(r"[^\w.-]+", "-", model_id)
49
+ return data_dir() / "eval" / "scores" / f"{task}.{name}.{key[:12]}.jsonl"
50
+
51
+
52
+ def read_jsonl(p: Path) -> list[dict[str, Any]]:
53
+ try:
54
+ with open(p, encoding="utf-8") as fh:
55
+ lines = fh.read().splitlines()
56
+ except FileNotFoundError:
57
+ return []
58
+ except (OSError, UnicodeDecodeError) as exc: # never "no labels": those labels exist
59
+ raise AutError(f"label: cannot read {p}: {exc}") from None
60
+ # None: a line cut by a crash mid-write
61
+ return [o for line in lines if (o := parse_record(line)) is not None]
62
+
63
+
64
+ def append_jsonl(p: Path, row: dict[str, Any]) -> None:
65
+ p.parent.mkdir(parents=True, exist_ok=True)
66
+ fd = os.open(p, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) # private, as write_jsonl's is
67
+ with os.fdopen(fd, "a", encoding="utf-8") as fh:
68
+ fh.write(json.dumps(row, sort_keys=True) + "\n")
69
+
70
+
71
+ def write_jsonl(p: Path, rows: list[dict[str, Any]]) -> None:
72
+ atomic_write_text(p, "".join(json.dumps(r, sort_keys=True) + "\n" for r in rows))
73
+
74
+
75
+ def gold_latest(task: str) -> dict[str, str]:
76
+ """unit_id -> gold label of the current task version; a later answer for a unit wins."""
77
+ ver, ids = TASKS[task].ver, {c for c, _d in TASKS[task].candidates}
78
+ return {r["unit_id"]: r["gold"] for r in read_jsonl(eval_path(task, "gold"))
79
+ if r.get("task_ver") == ver and r.get("gold") in ids}
80
+
81
+
82
+ def silver_by_unit(task: str) -> dict[str, dict[str, dict[str, float]]]:
83
+ """unit_id -> {teacher: dist} of the current task version."""
84
+ out: dict[str, dict[str, Any]] = defaultdict(dict)
85
+ for r in read_jsonl(eval_path(task, "silver")):
86
+ if r.get("task_ver") == TASKS[task].ver and isinstance(r.get("dist"), dict):
87
+ out[r["unit_id"]][r["teacher"]] = r["dist"]
88
+ return dict(out)
89
+
90
+
91
+ def stratum(teachers: dict[str, Any] | None) -> str:
92
+ """The silver stratum of a unit: the teachers' common top label, `disagree`, or
93
+ `unlabelled`."""
94
+ tops = {top_label(d) for d in (teachers or {}).values() if d}
95
+ return "unlabelled" if not tops else tops.pop() if len(tops) == 1 else "disagree"
96
+
97
+
98
+ def stratified(items: list[T], key: Callable[[T], object], seed: str) -> list[T]:
99
+ """Round-robin over the strata of `key`, each stratum shuffled: rare strata come up as often
100
+ as common ones. Deterministic for a seed, so a resumed sitting sees the same order."""
101
+ rng = random.Random(seed) # nosec B311 # a repeatable sample order, not security
102
+ groups: dict[object, list[T]] = defaultdict(list)
103
+ for x in items:
104
+ groups[key(x)].append(x)
105
+ keys = sorted(groups, key=str)
106
+ for k in keys:
107
+ rng.shuffle(groups[k])
108
+ rng.shuffle(keys)
109
+ return [x for row in itertools.zip_longest(*(groups[k] for k in keys)) for x in row
110
+ if x is not None]
111
+
112
+
113
+ def unit_day(req: Request) -> date:
114
+ return datetime.fromisoformat(req.ts).astimezone().date()
115
+
116
+
117
+ def _hash(key: str) -> int:
118
+ return int(hashlib.sha256(key.encode()).hexdigest(), 16)
119
+
120
+
121
+ def _group(req: Request) -> str:
122
+ """What must stay together in a split: the unit's local day. A title's ts is its first sight
123
+ inside the window read, which moves, so a title is its own group."""
124
+ return req.unit_id if req.unit == "aw_title" else unit_day(req).isoformat()
125
+
126
+
127
+ def final_test(req: Request) -> bool:
128
+ """The stable part of the days only `aut label eval --final` reads: no fold, tau, temperature,
129
+ encoder fit or choice sees them."""
130
+ return _hash("final|" + _group(req)) % 100 < FINAL_PCT
131
+
132
+
133
+ def fold(req: Request) -> int:
134
+ """The unit's fold, the same for every model and across runs. shortcut: whole days hash to a
135
+ fold, so folds are not stratified by label and a rare label can sit in few folds; upgrade by
136
+ assigning days to folds by their gold label mix once gold is large enough to care."""
137
+ return _hash("fold|" + _group(req)) % FOLDS
138
+
139
+
140
+ def in_encoder_train_split(req: Request) -> bool:
141
+ """The half of the days the encoder trains on; the other half fits its temperature and tau in
142
+ eval, so they are never fitted on the encoder's own training data."""
143
+ return _hash("encoder|" + _group(req)) % 2 == 1
144
+
145
+
146
+ def window_units(cfg: Config, task: str, days: int = LABEL_DAYS, src: Source | None = None,
147
+ today: date | None = None) -> dict[str, Request]:
148
+ """unit_id -> request for the task's units over the last `days` whole local days."""
149
+ if src is None:
150
+ cfg.require_pg("label (it reads the steer and episode rows)")
151
+ today = today or date.today()
152
+ reqs = units(task, src or Source(cfg), local_midnight(today - timedelta(days=days)),
153
+ local_midnight(today))
154
+ return {r.unit_id: r for r in reqs}