agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
"""The decision interface every label task uses, its calibration, and the stored label row.
|
|
2
|
+
|
|
3
|
+
A backend returns a raw distribution over the request's candidates. Calibration (temperature
|
|
4
|
+
scaling, or Platt scaling on a yes/no task, fitted per task and model on gold labels, and only
|
|
5
|
+
when it helps) turns it into the calibrated distribution, and a per-label threshold tau decides
|
|
6
|
+
whether the top answer counts: below tau the row is still written,
|
|
7
|
+
with below_tau = true, and readers treat it as unknown. "Unknown" is never a candidate.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import math
|
|
14
|
+
import os
|
|
15
|
+
import sys
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from datetime import date, datetime, timedelta, timezone
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any, Callable, Iterable, Iterator, Protocol, runtime_checkable
|
|
20
|
+
|
|
21
|
+
from ..config import config_path
|
|
22
|
+
from ..event import ActivityEvent, make_event_id
|
|
23
|
+
|
|
24
|
+
PROMPT_VER = 2 # bump when render_prompt() or the chat prompt changes; task_ver covers candidates/definitions
|
|
25
|
+
LETTERS = "ABCDEFGHIJ"
|
|
26
|
+
TAU_FLOOR = 0.5
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def local_midnight(d: date) -> datetime:
|
|
30
|
+
return datetime(d.year, d.month, d.day).astimezone()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def local_days(since: datetime, until: datetime) -> Iterator[date]:
|
|
34
|
+
"""The local dates from `since`'s up to, not including, `until`'s."""
|
|
35
|
+
d, end = since.astimezone().date(), until.astimezone().date()
|
|
36
|
+
while d < end:
|
|
37
|
+
yield d
|
|
38
|
+
d += timedelta(days=1)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def data_dir() -> Path:
|
|
42
|
+
"""Where label data that never leaves this machine lives (gold and silver under eval/)."""
|
|
43
|
+
base = os.environ.get("XDG_DATA_HOME", str(Path.home() / ".local" / "share"))
|
|
44
|
+
return Path(base) / "aut"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def choice_path() -> Path:
|
|
48
|
+
"""The chosen backend, model, temperature and tau per task (written by `aut label choose`)."""
|
|
49
|
+
return config_path().parent / "label.json"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(slots=True)
|
|
53
|
+
class Request:
|
|
54
|
+
task: str
|
|
55
|
+
task_ver: int
|
|
56
|
+
unit: str # steer | episode_day | aw_block | episode | aw_title
|
|
57
|
+
unit_id: str
|
|
58
|
+
ts: str # the unit's own time (ISO-8601): the label row's ts
|
|
59
|
+
facts: dict[str, Any] # redacted; only explicit private replay jobs retain the facts
|
|
60
|
+
question: str
|
|
61
|
+
decision: str # choice | boolean
|
|
62
|
+
candidates: list[dict[str, Any]] # [{"id", "desc"}]: the task's closed list
|
|
63
|
+
none_fits: str | None # the explicit out-of-list candidate ("other"), or None
|
|
64
|
+
rev: str | None = None # the unit's version, when a changed unit is re-asked (episode)
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def prompt_ver(self) -> str:
|
|
68
|
+
return f"{self.task}@{self.task_ver}/p{PROMPT_VER}"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(slots=True)
|
|
72
|
+
class Response:
|
|
73
|
+
dist: dict[str, float] # calibrated, over every candidate
|
|
74
|
+
chosen: str
|
|
75
|
+
prob: float
|
|
76
|
+
below_tau: bool
|
|
77
|
+
model_id: str
|
|
78
|
+
model_rev: str
|
|
79
|
+
prompt_ver: str
|
|
80
|
+
calib_ver: str
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass(slots=True)
|
|
84
|
+
class Choice:
|
|
85
|
+
"""One task's winner. The default is uncalibrated: T = 1 and no tau, so nothing passes."""
|
|
86
|
+
backend: str | None = None # None: nothing recorded, the model table decides
|
|
87
|
+
model_id: str | None = None
|
|
88
|
+
model_rev: str | None = None
|
|
89
|
+
temperature: float = 1.0
|
|
90
|
+
platt: dict[str, Any] | None = None # {"yes", "a", "b"}: replaces the temperature
|
|
91
|
+
# label -> tau; a missing or null label never passes
|
|
92
|
+
tau: dict[str, float | None] = field(default_factory=dict)
|
|
93
|
+
calib_ver: str = "none"
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class Backend(Protocol):
|
|
97
|
+
"""What a decision reads. model_rev and size_mb may only be final after the first distribution
|
|
98
|
+
call, and model_rev goes into the label row's event_id: read them after scoring. A backend may
|
|
99
|
+
add prepare() (find the weights and fix model_rev without loading; the bake-off calls it first,
|
|
100
|
+
because saved scores are keyed on model_rev), peak_memory_mb(), close(), and a `protocol`
|
|
101
|
+
string for the saved-scores key."""
|
|
102
|
+
backend: str # what `aut label choose` writes, what make_backend reads
|
|
103
|
+
model_id: str
|
|
104
|
+
model_rev: str
|
|
105
|
+
size_mb: float | None
|
|
106
|
+
|
|
107
|
+
def distribution(self, req: Request) -> dict[str, float]: ...
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@runtime_checkable
|
|
111
|
+
class NoteBackend(Backend, Protocol):
|
|
112
|
+
"""A backend that can also write free text, for the private nightly note."""
|
|
113
|
+
|
|
114
|
+
def generate(self, prompt: str, max_tokens: int = 300) -> str: ...
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def read_choices() -> dict[str, Any]:
|
|
118
|
+
"""label.json; {} when there is none yet. Raises OSError or ValueError when it is unusable."""
|
|
119
|
+
try:
|
|
120
|
+
obj = json.loads(choice_path().read_text())
|
|
121
|
+
except FileNotFoundError:
|
|
122
|
+
return {}
|
|
123
|
+
if not isinstance(obj, dict) or not isinstance(obj.get("tasks", {}), dict):
|
|
124
|
+
raise ValueError("unexpected shape")
|
|
125
|
+
return obj
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _is_number(x: object) -> bool:
|
|
129
|
+
return isinstance(x, (int, float)) and not isinstance(x, bool)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _check_entry(task: str, c: object) -> None:
|
|
133
|
+
"""Raises ValueError for a label.json entry whose values decide() could not use."""
|
|
134
|
+
if not isinstance(c, dict):
|
|
135
|
+
raise ValueError(f"its entry for {task} is not a table")
|
|
136
|
+
t, tau = c.get("temperature", 1.0), c.get("tau", {})
|
|
137
|
+
if not _is_number(t) or t <= 0:
|
|
138
|
+
raise ValueError(f"the temperature of {task} is not a positive number")
|
|
139
|
+
if not isinstance(tau, dict) or not all(v is None or _is_number(v) for v in tau.values()):
|
|
140
|
+
raise ValueError(f"the tau of {task} is not a table of numbers")
|
|
141
|
+
platt = c.get("platt")
|
|
142
|
+
if platt is not None and not (isinstance(platt, dict) and isinstance(platt.get("yes"), str)
|
|
143
|
+
and _is_number(platt.get("a")) and _is_number(platt.get("b"))):
|
|
144
|
+
raise ValueError(f"the platt of {task} is not a table of yes, a and b")
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def load_choice(task: str, task_ver: int | None = None) -> Choice:
|
|
148
|
+
try:
|
|
149
|
+
c = read_choices().get("tasks", {}).get(task)
|
|
150
|
+
if c is not None:
|
|
151
|
+
_check_entry(task, c)
|
|
152
|
+
if task_ver is not None and c.get("prompt_ver") != f"{task}@{task_ver}/p{PROMPT_VER}":
|
|
153
|
+
raise ValueError(f"the calibration of {task} belongs to another prompt version")
|
|
154
|
+
except (OSError, ValueError) as exc:
|
|
155
|
+
print(f"aut: warning: {choice_path()} is unusable ({exc}); {task} runs uncalibrated, so "
|
|
156
|
+
"none of its labels pass tau. Fix it or move it aside and re-run `aut label choose`.", file=sys.stderr)
|
|
157
|
+
return Choice()
|
|
158
|
+
if c is None:
|
|
159
|
+
return Choice()
|
|
160
|
+
return Choice(**{k: c[k] for k in Choice.__slots__ if k in c})
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def render_prompt(req: Request, order: list[str]) -> str:
|
|
164
|
+
"""The prompt for one candidate order: candidates as letters, the facts, a one-letter answer."""
|
|
165
|
+
desc = {c["id"]: c["desc"] for c in req.candidates}
|
|
166
|
+
lines = [req.question, "", "Facts:"]
|
|
167
|
+
lines += [f"- {k}: {v if isinstance(v, str) else json.dumps(v, ensure_ascii=False)}"
|
|
168
|
+
for k, v in req.facts.items()]
|
|
169
|
+
lines += ["", "Options:"] + [f"{LETTERS[i]}. {c}: {desc[c]}" for i, c in enumerate(order)]
|
|
170
|
+
lines += ["", f"Answer with one letter ({', '.join(LETTERS[:len(order)])}) and nothing else."]
|
|
171
|
+
return "\n".join(lines)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def softmax(xs: list[float]) -> list[float]:
|
|
175
|
+
m = max(xs)
|
|
176
|
+
e = [math.exp(x - m) for x in xs]
|
|
177
|
+
s = sum(e)
|
|
178
|
+
return [v / s for v in e]
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def top_label(dist: dict[str, float]) -> str:
|
|
182
|
+
return max(dist, key=lambda label: dist[label])
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def letter_distribution(letter_logits: Callable[[str, list[str]], list[float]],
|
|
186
|
+
req: Request) -> dict[str, float]:
|
|
187
|
+
"""letter_logits(prompt, letters) -> one logit per letter at the answer position. Scores the
|
|
188
|
+
original and the reversed candidate order and averages the two (letter position bias)."""
|
|
189
|
+
ids = [c["id"] for c in req.candidates]
|
|
190
|
+
if not 1 <= len(ids) <= len(LETTERS):
|
|
191
|
+
raise ValueError(f"{req.task}: {len(ids)} candidates, at most {len(LETTERS)}")
|
|
192
|
+
out = dict.fromkeys(ids, 0.0)
|
|
193
|
+
orders = (ids, ids[::-1])
|
|
194
|
+
for order in orders:
|
|
195
|
+
for c, p in zip(order, softmax(letter_logits(render_prompt(req, order),
|
|
196
|
+
list(LETTERS[:len(order)])))):
|
|
197
|
+
out[c] += p / len(orders)
|
|
198
|
+
return out
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def temper(dist: dict[str, float], t: float) -> dict[str, float]:
|
|
202
|
+
"""Temperature scaling of a distribution: softmax(log p / t)."""
|
|
203
|
+
keys = list(dist)
|
|
204
|
+
return dict(zip(keys, softmax([math.log(max(dist[k], 1e-12)) / t for k in keys])))
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def fit_temperature(dists: list[dict[str, Any]], golds: list[str], lo: float = 0.05,
|
|
208
|
+
hi: float = 20.0) -> float:
|
|
209
|
+
"""The T that minimises the gold labels' negative log likelihood (golden section on log T;
|
|
210
|
+
the NLL of temperature scaling is unimodal in it). 1.0 without data."""
|
|
211
|
+
if not dists:
|
|
212
|
+
return 1.0
|
|
213
|
+
|
|
214
|
+
def nll(lt: float) -> float:
|
|
215
|
+
t = math.exp(lt)
|
|
216
|
+
return -sum(math.log(max(temper(d, t)[g], 1e-12)) for d, g in zip(dists, golds))
|
|
217
|
+
|
|
218
|
+
a, b, r = math.log(lo), math.log(hi), (math.sqrt(5) - 1) / 2
|
|
219
|
+
for _ in range(80):
|
|
220
|
+
c, d = b - r * (b - a), a + r * (b - a)
|
|
221
|
+
if nll(c) < nll(d):
|
|
222
|
+
b = d
|
|
223
|
+
else:
|
|
224
|
+
a = c
|
|
225
|
+
return round(math.exp((a + b) / 2), 4)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _sigmoid(x: float) -> float:
|
|
229
|
+
return 1 / (1 + math.exp(-max(min(x, 35.0), -35.0)))
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _log_odds(p: float) -> float:
|
|
233
|
+
p = min(max(p, 1e-6), 1 - 1e-6)
|
|
234
|
+
return math.log(p / (1 - p))
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def platt_scale(dist: dict[str, float], platt: dict[str, Any]) -> dict[str, float]:
|
|
238
|
+
"""Platt scaling of a yes/no distribution: sigmoid(a * log-odds of "yes" + b)."""
|
|
239
|
+
q = _sigmoid(platt["a"] * _log_odds(dist[platt["yes"]]) + platt["b"])
|
|
240
|
+
return {k: q if k == platt["yes"] else 1 - q for k in dist}
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def calibrate(raw: dict[str, float], temperature: float = 1.0,
|
|
244
|
+
platt: dict[str, Any] | None = None) -> dict[str, float]:
|
|
245
|
+
return platt_scale(raw, platt) if platt else temper(raw, temperature)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def fit_platt(dists: list[dict[str, float]], golds: list[str], yes: str,
|
|
249
|
+
l2: float = 1e-3) -> dict[str, Any]:
|
|
250
|
+
"""The a and b that minimise the gold labels' NLL on the log-odds of `yes` (damped Newton; a
|
|
251
|
+
small pull towards a = 1, b = 0 keeps separable data finite). The identity without data."""
|
|
252
|
+
xy = [(_log_odds(d[yes]), float(g == yes)) for d, g in zip(dists, golds)]
|
|
253
|
+
|
|
254
|
+
def loss(a: float, b: float) -> float:
|
|
255
|
+
return l2 / 2 * ((a - 1) ** 2 + b ** 2) - sum(
|
|
256
|
+
math.log(max(q if y else 1 - q, 1e-12))
|
|
257
|
+
for x, y in xy for q in [_sigmoid(a * x + b)])
|
|
258
|
+
|
|
259
|
+
a, b = 1.0, 0.0
|
|
260
|
+
for _ in range(50 if xy else 0):
|
|
261
|
+
ga, gb, haa, hab, hbb = l2 * (a - 1), l2 * b, l2, 0.0, l2
|
|
262
|
+
for x, y in xy:
|
|
263
|
+
q = _sigmoid(a * x + b)
|
|
264
|
+
w = q * (1 - q)
|
|
265
|
+
ga, gb = ga + (q - y) * x, gb + q - y
|
|
266
|
+
haa, hab, hbb = haa + w * x * x, hab + w * x, hbb + w
|
|
267
|
+
det = haa * hbb - hab * hab
|
|
268
|
+
da, db = (hbb * ga - hab * gb) / det, (haa * gb - hab * ga) / det
|
|
269
|
+
step, base = 1.0, loss(a, b)
|
|
270
|
+
while step > 1e-4 and loss(a - step * da, b - step * db) > base:
|
|
271
|
+
step /= 2
|
|
272
|
+
a, b = a - step * da, b - step * db
|
|
273
|
+
if abs(step * da) + abs(step * db) < 1e-6:
|
|
274
|
+
break
|
|
275
|
+
return {"yes": yes, "a": round(a, 4), "b": round(b, 4)}
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def fit_tau(items: list[tuple[str, float, str]], precision: float = 0.90,
|
|
279
|
+
floor: float = TAU_FLOOR, recall: float = 0.0) -> dict[str, float | None]:
|
|
280
|
+
"""items: (chosen, p_cal, gold). Per chosen label: of the cuts between distinct p, no lower than
|
|
281
|
+
`floor`, the one with the best F1 among those that reach `precision` and `recall` on the items
|
|
282
|
+
chosen as that label with p >= it, the lowest on a tie. None when no cut does: the floors are
|
|
283
|
+
never relaxed, and the label never passes."""
|
|
284
|
+
by: dict[str, list[tuple[float, bool]]] = {}
|
|
285
|
+
true: dict[str, int] = {}
|
|
286
|
+
for chosen, p, gold in items:
|
|
287
|
+
by.setdefault(chosen, []).append((p, chosen == gold))
|
|
288
|
+
true[gold] = true.get(gold, 0) + 1
|
|
289
|
+
out: dict[str, float | None] = {}
|
|
290
|
+
for label, rows in by.items():
|
|
291
|
+
rows.sort(key=lambda r: -r[0])
|
|
292
|
+
n_true = true.get(label, 0)
|
|
293
|
+
best, top, ok = None, -1.0, 0
|
|
294
|
+
for i, (p, hit) in enumerate(rows):
|
|
295
|
+
if p < floor:
|
|
296
|
+
break
|
|
297
|
+
ok += hit
|
|
298
|
+
below = rows[i + 1][0] if i + 1 < len(rows) else None
|
|
299
|
+
f1 = 2 * ok / (i + 1 + n_true)
|
|
300
|
+
if (below is None or below < p) and ok / (i + 1) >= precision \
|
|
301
|
+
and ok / max(n_true, 1) >= recall and f1 >= top:
|
|
302
|
+
best, top = (floor if below is not None and below < floor else p), f1
|
|
303
|
+
out[label] = best
|
|
304
|
+
return out
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def decide(req: Request, raw: dict[str, float], choice: Choice, model_id: str,
|
|
308
|
+
model_rev: str) -> Response:
|
|
309
|
+
dist = calibrate(raw, choice.temperature, choice.platt)
|
|
310
|
+
chosen = max(dist, key=dist.__getitem__)
|
|
311
|
+
tau = choice.tau.get(chosen)
|
|
312
|
+
return Response(dist=dist, chosen=chosen, prob=dist[chosen],
|
|
313
|
+
below_tau=tau is None or dist[chosen] < tau, model_id=model_id,
|
|
314
|
+
model_rev=model_rev, prompt_ver=req.prompt_ver, calib_ver=choice.calib_ver)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def label_event(req: Request, res: Response, labelled_at: str | None = None) -> ActivityEvent:
|
|
318
|
+
"""The activity_event row of one decision: enums, numbers and versions, never the facts.
|
|
319
|
+
Same unit, task version, model and calibration -> same id, so a rerun is a no-op at the sink;
|
|
320
|
+
a new model, prompt or calibration (or a changed episode, req.rev) adds a row and never
|
|
321
|
+
overwrites one. labelled_at (UTC, fixed width) is the run's start: agents.label_latest takes
|
|
322
|
+
the newest one, so a later calibrated row wins over an earlier uncalibrated one."""
|
|
323
|
+
labelled_at = labelled_at or datetime.now(timezone.utc).isoformat(timespec="milliseconds")
|
|
324
|
+
return ActivityEvent(
|
|
325
|
+
event_id=make_event_id("label", req.task, req.task_ver, req.unit_id, res.model_id,
|
|
326
|
+
res.model_rev, res.prompt_ver, res.calib_ver,
|
|
327
|
+
*((req.rev,) if req.rev else ())),
|
|
328
|
+
ts=req.ts, agent="label", kind="label", session_id=None, turn_id=None,
|
|
329
|
+
attrs={"task": req.task, "task_ver": req.task_ver, "unit": req.unit,
|
|
330
|
+
"unit_id": req.unit_id, "label": res.chosen, "p": round(res.prob, 4),
|
|
331
|
+
"dist": {k: round(v, 4) for k, v in res.dist.items()},
|
|
332
|
+
"below_tau": res.below_tau, "model_id": res.model_id, "model_rev": res.model_rev,
|
|
333
|
+
"prompt_ver": res.prompt_ver, "calib_ver": res.calib_ver,
|
|
334
|
+
"labelled_at": labelled_at})
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def current_labels(rows: Iterable[tuple[Any, ...]]) -> list[dict[str, Any]]:
|
|
338
|
+
"""The one current label per (task, unit, unit_id) of (event_id, ts, attrs) rows, picked as
|
|
339
|
+
agents.label_latest picks it: newest task_ver, then newest labelled_at (a row without one
|
|
340
|
+
loses), then event_id. A row with no below_tau counts as below: readers treat it as unknown.
|
|
341
|
+
Working estimates have separate consumers and never enter calibrated selection."""
|
|
342
|
+
best: dict[tuple[Any, ...], tuple[Any, ...]] = {}
|
|
343
|
+
for eid, ts, a in rows:
|
|
344
|
+
a = a or {}
|
|
345
|
+
if str(a.get("task") or "").startswith("working."):
|
|
346
|
+
continue
|
|
347
|
+
key = (a.get("task"), a.get("unit"), a.get("unit_id"))
|
|
348
|
+
rank = (int(a.get("task_ver") or 0), a.get("labelled_at") or "", eid)
|
|
349
|
+
if key not in best or rank > best[key][0]:
|
|
350
|
+
below = a.get("below_tau")
|
|
351
|
+
best[key] = (rank, {"task": key[0], "unit": key[1], "unit_id": key[2], "ts": ts,
|
|
352
|
+
"label": a.get("label"), "p": a.get("p"),
|
|
353
|
+
"below_tau": True if below is None else bool(below),
|
|
354
|
+
"model_id": a.get("model_id"), "task_ver": rank[0]})
|
|
355
|
+
return [v for _r, v in best.values()]
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""Opt-in local outcome catch-up with acknowledged, resumable private coverage progress."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
import time
|
|
8
|
+
from contextlib import nullcontext
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import TYPE_CHECKING, Any, Callable
|
|
11
|
+
|
|
12
|
+
from ..config import AutError
|
|
13
|
+
from ..event import ActivityEvent
|
|
14
|
+
from ..state import LockBusy, _locked, atomic_write_text
|
|
15
|
+
from .backends import BackendUnavailable
|
|
16
|
+
from .core import Choice, Request, data_dir, decide, label_event
|
|
17
|
+
from .local import LocalBackend, decorate
|
|
18
|
+
from .outcome_coverage import coverage_key, pending_outcomes
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from .run import _Run
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _load(path: Path, machine: str) -> tuple[set[str], bool]:
|
|
25
|
+
try:
|
|
26
|
+
value = json.loads(path.read_text())
|
|
27
|
+
except FileNotFoundError:
|
|
28
|
+
return set(), False
|
|
29
|
+
except (OSError, ValueError) as exc:
|
|
30
|
+
raise AutError("label: outcome coverage progress is unreadable; it was not reset") from exc
|
|
31
|
+
if (not isinstance(value, dict) or value.get("version") != 1
|
|
32
|
+
or value.get("machine") != machine or type(value.get("next_mature")) is not bool
|
|
33
|
+
or not isinstance(value.get("acknowledged"), list)
|
|
34
|
+
or any(not isinstance(k, str) or not re.fullmatch(r"[a-f0-9]{64}", k)
|
|
35
|
+
for k in value["acknowledged"])):
|
|
36
|
+
raise AutError("label: invalid outcome coverage progress; it was not reset")
|
|
37
|
+
return set(value["acknowledged"]), value["next_mature"]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def covered_outcomes(run: _Run, requests: list[Request], backend: LocalBackend, choice: Choice,
|
|
41
|
+
report: dict[str, Any], batch_size: int,
|
|
42
|
+
deliver: Callable[[str, list[ActivityEvent]], Any]) -> None:
|
|
43
|
+
path = data_dir() / "label" / "outcome-coverage.json"
|
|
44
|
+
try:
|
|
45
|
+
with nullcontext() if run.dry_run else _locked(path, blocking=False):
|
|
46
|
+
done, next_mature = _load(path, run.cfg.machine)
|
|
47
|
+
pending = pending_outcomes(requests, done, backend.policy_id, start_mature=next_mature)
|
|
48
|
+
current = run.src.outcome_revisions([r for key, r in pending])
|
|
49
|
+
eligible = [(key, r) for key, r in pending if current.get(r.unit_id) == r.rev]
|
|
50
|
+
report["skipped_stale_outcome"] += len(pending) - len(eligible)
|
|
51
|
+
backend.prepare_requests([r for key, r in eligible], run.deadline)
|
|
52
|
+
report["provisional"] = 1
|
|
53
|
+
batch: list[tuple[str, Request, ActivityEvent]] = []
|
|
54
|
+
|
|
55
|
+
def flush() -> None:
|
|
56
|
+
nonlocal next_mature
|
|
57
|
+
revisions = run.src.outcome_revisions([r for key, r, ev in batch])
|
|
58
|
+
valid = [(key, r, ev) for key, r, ev in batch
|
|
59
|
+
if revisions.get(r.unit_id) == r.rev]
|
|
60
|
+
report["skipped_stale_outcome"] += len(batch) - len(valid)
|
|
61
|
+
if valid and not run.dry_run:
|
|
62
|
+
deliver(run.dsn, [ev for key, r, ev in valid])
|
|
63
|
+
# Unknown is an acknowledged estimate too. Never retry it as success hunting.
|
|
64
|
+
done.update(key for key, r, ev in valid)
|
|
65
|
+
next_mature = not coverage_key(valid[-1][1], backend.policy_id)[1]
|
|
66
|
+
atomic_write_text(path, json.dumps({"version": 1, "machine": run.cfg.machine,
|
|
67
|
+
"acknowledged": sorted(done), "next_mature": next_mature}, sort_keys=True))
|
|
68
|
+
for key, r, ev in valid:
|
|
69
|
+
report["rows"] += 1
|
|
70
|
+
report["below_tau"] += ev.attrs["below_tau"]
|
|
71
|
+
report["labels"][ev.attrs["label"]] += 1
|
|
72
|
+
batch.clear()
|
|
73
|
+
|
|
74
|
+
try:
|
|
75
|
+
for key, request in eligible:
|
|
76
|
+
if time.monotonic() >= run.deadline:
|
|
77
|
+
report["cut"] = 1
|
|
78
|
+
break
|
|
79
|
+
raw = backend.distribution(request)
|
|
80
|
+
result = decide(request, raw, choice, backend.model_id, backend.model_rev)
|
|
81
|
+
event = decorate(label_event(request, result, run.labelled_at), request,
|
|
82
|
+
backend, result.prob)
|
|
83
|
+
event.machine = run.cfg.machine
|
|
84
|
+
batch.append((key, request, event))
|
|
85
|
+
if len(batch) >= batch_size:
|
|
86
|
+
flush()
|
|
87
|
+
except BackendUnavailable:
|
|
88
|
+
flush() # Preserve completed estimates when a later model batch exhausts its budget.
|
|
89
|
+
raise
|
|
90
|
+
flush()
|
|
91
|
+
except LockBusy as exc:
|
|
92
|
+
raise BackendUnavailable("label: outcome coverage is already running") from exc
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""The label files that never leave this machine, and the unit order the teacher and the gold page
|
|
2
|
+
share.
|
|
3
|
+
|
|
4
|
+
Under core.data_dir() / "eval": <task>.silver.jsonl (teacher distributions: unit_id, teacher,
|
|
5
|
+
dist, date, task_ver), <task>.jsonl (gold: unit_id, gold, labeller, date, task_ver),
|
|
6
|
+
<task>.eval.jsonl (one bake-off record per backend, numbers only), <task>.final.jsonl (results on
|
|
7
|
+
the final test days) and scores/ (one file per task, model, revision and prompt: unit_id and raw
|
|
8
|
+
dist). No row holds text: a unit's text is rebuilt from this machine's logs (tasks.units) whenever
|
|
9
|
+
it is needed.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import itertools
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import random
|
|
19
|
+
import re
|
|
20
|
+
from collections import defaultdict
|
|
21
|
+
from datetime import date, datetime, timedelta
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any, Callable, TypeVar
|
|
24
|
+
|
|
25
|
+
from ..config import LABEL_DAYS, AutError, Config
|
|
26
|
+
from ..state import atomic_write_text, parse_record
|
|
27
|
+
from .core import Request, data_dir, local_midnight, top_label
|
|
28
|
+
from .tasks import TASKS, Source, units
|
|
29
|
+
|
|
30
|
+
T = TypeVar("T")
|
|
31
|
+
# Gold per label, (floor, target), as the gold page shows them. AW titles: 150 per class.
|
|
32
|
+
FLOORS = {"work_category": (30, 50), "steer_intent": (30, 50), "soft_outcome": (30, 30),
|
|
33
|
+
"aw_title_relevance": (150, 150), "writing_activity": (30, 50)}
|
|
34
|
+
FOLDS = 5
|
|
35
|
+
FINAL_PCT = 20 # share of the days kept for `aut label eval --final`
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def eval_path(task: str, kind: str) -> Path:
|
|
39
|
+
"""kind: silver | gold | eval | final."""
|
|
40
|
+
return data_dir() / "eval" / (f"{task}.jsonl" if kind == "gold" else f"{task}.{kind}.jsonl")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def scores_path(task: str, model_id: str, model_rev: str, prompt_ver: str,
|
|
44
|
+
protocol: str = "") -> Path:
|
|
45
|
+
"""The saved raw scores of one task, model, revision, prompt and decision protocol."""
|
|
46
|
+
parts = (task, model_id, model_rev, prompt_ver) + ((protocol,) if protocol else ())
|
|
47
|
+
key = hashlib.sha256("\0".join(parts).encode()).hexdigest()
|
|
48
|
+
name = re.sub(r"[^\w.-]+", "-", model_id)
|
|
49
|
+
return data_dir() / "eval" / "scores" / f"{task}.{name}.{key[:12]}.jsonl"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def read_jsonl(p: Path) -> list[dict[str, Any]]:
|
|
53
|
+
try:
|
|
54
|
+
with open(p, encoding="utf-8") as fh:
|
|
55
|
+
lines = fh.read().splitlines()
|
|
56
|
+
except FileNotFoundError:
|
|
57
|
+
return []
|
|
58
|
+
except (OSError, UnicodeDecodeError) as exc: # never "no labels": those labels exist
|
|
59
|
+
raise AutError(f"label: cannot read {p}: {exc}") from None
|
|
60
|
+
# None: a line cut by a crash mid-write
|
|
61
|
+
return [o for line in lines if (o := parse_record(line)) is not None]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def append_jsonl(p: Path, row: dict[str, Any]) -> None:
|
|
65
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
66
|
+
fd = os.open(p, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) # private, as write_jsonl's is
|
|
67
|
+
with os.fdopen(fd, "a", encoding="utf-8") as fh:
|
|
68
|
+
fh.write(json.dumps(row, sort_keys=True) + "\n")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def write_jsonl(p: Path, rows: list[dict[str, Any]]) -> None:
|
|
72
|
+
atomic_write_text(p, "".join(json.dumps(r, sort_keys=True) + "\n" for r in rows))
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def gold_latest(task: str) -> dict[str, str]:
|
|
76
|
+
"""unit_id -> gold label of the current task version; a later answer for a unit wins."""
|
|
77
|
+
ver, ids = TASKS[task].ver, {c for c, _d in TASKS[task].candidates}
|
|
78
|
+
return {r["unit_id"]: r["gold"] for r in read_jsonl(eval_path(task, "gold"))
|
|
79
|
+
if r.get("task_ver") == ver and r.get("gold") in ids}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def silver_by_unit(task: str) -> dict[str, dict[str, dict[str, float]]]:
|
|
83
|
+
"""unit_id -> {teacher: dist} of the current task version."""
|
|
84
|
+
out: dict[str, dict[str, Any]] = defaultdict(dict)
|
|
85
|
+
for r in read_jsonl(eval_path(task, "silver")):
|
|
86
|
+
if r.get("task_ver") == TASKS[task].ver and isinstance(r.get("dist"), dict):
|
|
87
|
+
out[r["unit_id"]][r["teacher"]] = r["dist"]
|
|
88
|
+
return dict(out)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def stratum(teachers: dict[str, Any] | None) -> str:
|
|
92
|
+
"""The silver stratum of a unit: the teachers' common top label, `disagree`, or
|
|
93
|
+
`unlabelled`."""
|
|
94
|
+
tops = {top_label(d) for d in (teachers or {}).values() if d}
|
|
95
|
+
return "unlabelled" if not tops else tops.pop() if len(tops) == 1 else "disagree"
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def stratified(items: list[T], key: Callable[[T], object], seed: str) -> list[T]:
|
|
99
|
+
"""Round-robin over the strata of `key`, each stratum shuffled: rare strata come up as often
|
|
100
|
+
as common ones. Deterministic for a seed, so a resumed sitting sees the same order."""
|
|
101
|
+
rng = random.Random(seed) # nosec B311 # a repeatable sample order, not security
|
|
102
|
+
groups: dict[object, list[T]] = defaultdict(list)
|
|
103
|
+
for x in items:
|
|
104
|
+
groups[key(x)].append(x)
|
|
105
|
+
keys = sorted(groups, key=str)
|
|
106
|
+
for k in keys:
|
|
107
|
+
rng.shuffle(groups[k])
|
|
108
|
+
rng.shuffle(keys)
|
|
109
|
+
return [x for row in itertools.zip_longest(*(groups[k] for k in keys)) for x in row
|
|
110
|
+
if x is not None]
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def unit_day(req: Request) -> date:
|
|
114
|
+
return datetime.fromisoformat(req.ts).astimezone().date()
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _hash(key: str) -> int:
|
|
118
|
+
return int(hashlib.sha256(key.encode()).hexdigest(), 16)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _group(req: Request) -> str:
|
|
122
|
+
"""What must stay together in a split: the unit's local day. A title's ts is its first sight
|
|
123
|
+
inside the window read, which moves, so a title is its own group."""
|
|
124
|
+
return req.unit_id if req.unit == "aw_title" else unit_day(req).isoformat()
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def final_test(req: Request) -> bool:
|
|
128
|
+
"""The stable part of the days only `aut label eval --final` reads: no fold, tau, temperature,
|
|
129
|
+
encoder fit or choice sees them."""
|
|
130
|
+
return _hash("final|" + _group(req)) % 100 < FINAL_PCT
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def fold(req: Request) -> int:
|
|
134
|
+
"""The unit's fold, the same for every model and across runs. shortcut: whole days hash to a
|
|
135
|
+
fold, so folds are not stratified by label and a rare label can sit in few folds; upgrade by
|
|
136
|
+
assigning days to folds by their gold label mix once gold is large enough to care."""
|
|
137
|
+
return _hash("fold|" + _group(req)) % FOLDS
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def in_encoder_train_split(req: Request) -> bool:
|
|
141
|
+
"""The half of the days the encoder trains on; the other half fits its temperature and tau in
|
|
142
|
+
eval, so they are never fitted on the encoder's own training data."""
|
|
143
|
+
return _hash("encoder|" + _group(req)) % 2 == 1
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def window_units(cfg: Config, task: str, days: int = LABEL_DAYS, src: Source | None = None,
|
|
147
|
+
today: date | None = None) -> dict[str, Request]:
|
|
148
|
+
"""unit_id -> request for the task's units over the last `days` whole local days."""
|
|
149
|
+
if src is None:
|
|
150
|
+
cfg.require_pg("label (it reads the steer and episode rows)")
|
|
151
|
+
today = today or date.today()
|
|
152
|
+
reqs = units(task, src or Source(cfg), local_midnight(today - timedelta(days=days)),
|
|
153
|
+
local_midnight(today))
|
|
154
|
+
return {r.unit_id: r for r in reqs}
|