agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""`aut label review [--task T]`: label gold answers one unit at a time on a local page.
|
|
2
|
+
|
|
3
|
+
Served from 127.0.0.1 on a random port under a random token, like `aut episodes review`. Each
|
|
4
|
+
item shows its local, redacted text and the candidates; keys 1-9 answer, s skips, u undoes, q
|
|
5
|
+
ends. The teachers' (silver) answers show only after the item is answered. Items come stratified
|
|
6
|
+
by silver label (rare labels and teacher disagreement come up as often as common ones), and the
|
|
7
|
+
page shows gold per label against the floor and target. Answers append to <task>.jsonl at once
|
|
8
|
+
(unit_id, gold, labeller, date, task_ver; never text), so a sitting can stop any time and the
|
|
9
|
+
next one resumes with what is left.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import html
|
|
15
|
+
import json
|
|
16
|
+
from collections import Counter
|
|
17
|
+
from datetime import date, datetime, timezone
|
|
18
|
+
from typing import Any, Callable
|
|
19
|
+
|
|
20
|
+
from .. import local_page
|
|
21
|
+
from ..config import LABEL_DAYS, Config
|
|
22
|
+
from .core import Request, top_label
|
|
23
|
+
from .evalset import (FLOORS, append_jsonl, eval_path, gold_latest, read_jsonl,
|
|
24
|
+
silver_by_unit, stratified, stratum, window_units, write_jsonl)
|
|
25
|
+
from .tasks import TASKS, Source
|
|
26
|
+
|
|
27
|
+
LABELLER = "human"
|
|
28
|
+
_KEYS = "123456789"
|
|
29
|
+
_CSS = """pre { white-space: pre-wrap; word-break: break-word; background: transparent;
|
|
30
|
+
border: 1px solid var(--line); border-radius: 6px; padding: .5rem; margin: .2rem 0 .6rem; }
|
|
31
|
+
.mut { color: var(--mut); font-size: 13px; } dt { font-weight: 600; }
|
|
32
|
+
button { font: inherit; display: block; width: 100%; text-align: left; margin: .25rem 0;
|
|
33
|
+
padding: .45rem .6rem; border: 1px solid var(--line); border-radius: 6px;
|
|
34
|
+
background: transparent; color: var(--fg); cursor: pointer; }
|
|
35
|
+
button:focus-visible { outline: 2px solid var(--acc); }
|
|
36
|
+
.row button { display: inline-block; width: auto; margin-right: .4rem; }
|
|
37
|
+
table { border-collapse: collapse; font-size: 13px; } td { padding: 0 .6rem 0 0; }
|
|
38
|
+
.last { border-left: 3px solid var(--acc); padding-left: .6rem; }"""
|
|
39
|
+
_JS = """document.addEventListener('keydown', e => {
|
|
40
|
+
if (e.metaKey || e.ctrlKey || e.altKey) return;
|
|
41
|
+
const b = document.querySelector('[data-key="' + e.key.toLowerCase() + '"]');
|
|
42
|
+
if (b) { e.preventDefault(); b.click(); } });"""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _facts(facts: dict[str, Any]) -> str:
|
|
46
|
+
e = html.escape
|
|
47
|
+
parts = ["<dl>"]
|
|
48
|
+
for k, v in facts.items():
|
|
49
|
+
if isinstance(v, str) and v:
|
|
50
|
+
parts.append(f"<dt>{e(k)}</dt><dd><pre>{e(v)}</pre></dd>")
|
|
51
|
+
elif isinstance(v, list) and v and all(isinstance(x, str) for x in v):
|
|
52
|
+
parts.append(f"<dt>{e(k)}</dt><dd>" + "".join(f"<pre>{e(x)}</pre>" for x in v)
|
|
53
|
+
+ "</dd>")
|
|
54
|
+
elif v not in (None, "", [], {}):
|
|
55
|
+
parts.append(f"<dt>{e(k)}</dt><dd>{e(json.dumps(v, ensure_ascii=False))}</dd>")
|
|
56
|
+
return "".join(parts) + "</dl>"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _progress(task: str, gold: dict[str, str]) -> str:
|
|
60
|
+
floor, target = FLOORS[task]
|
|
61
|
+
n = Counter(gold.values())
|
|
62
|
+
rows = "".join(f"<tr><td>{html.escape(c)}</td><td>{n[c]} / {floor}</td>"
|
|
63
|
+
f"<td class=mut>{'floor reached' if n[c] >= floor else ''}</td></tr>"
|
|
64
|
+
for c, _d in TASKS[task].candidates)
|
|
65
|
+
return (f"<details><summary>Gold per label (floor {floor}, target {target})</summary>"
|
|
66
|
+
f"<table>{rows}</table></details>")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def render(task: str, token: str, item: Request | None, gold: dict[str, Any], last: dict[str, Any] | None,
|
|
70
|
+
left: int, counts: Counter[str]) -> str:
|
|
71
|
+
e = html.escape
|
|
72
|
+
head = (local_page.head("Gold labels", "52rem", _CSS) + f"<h1>{e(task)}</h1>"
|
|
73
|
+
"<p class=mut>Private page on this machine only. Only your answer (a label and the "
|
|
74
|
+
f"unit id) is saved, on this machine. This session: {counts['answered']} answered, "
|
|
75
|
+
f"{counts['skipped']} skipped; {left} left.</p>")
|
|
76
|
+
parts = [head]
|
|
77
|
+
if last:
|
|
78
|
+
said = ", ".join(f"{e(t)}: {e(lab)} ({p:.0%})" for t, lab, p in last["teachers"]) \
|
|
79
|
+
or "no teacher label"
|
|
80
|
+
parts.append(f"<p class=last>Last item: you said <b>{e(last['gold'])}</b>; teachers "
|
|
81
|
+
f"said {said}.</p>")
|
|
82
|
+
parts.append(f'<form method="post" action="/{e(token)}">')
|
|
83
|
+
if item is None:
|
|
84
|
+
parts.append("<p>Nothing left to label in this window.</p>")
|
|
85
|
+
else:
|
|
86
|
+
parts.append(f'<input type=hidden name=u value="{e(item.unit_id)}">'
|
|
87
|
+
f"<p><b>{e(item.question)}</b> <span class=mut>({e(item.unit)})</span></p>"
|
|
88
|
+
+ _facts(item.facts))
|
|
89
|
+
for i, c in enumerate(item.candidates):
|
|
90
|
+
k = _KEYS[i] if i < len(_KEYS) else ""
|
|
91
|
+
parts.append(f'<button name=a value="{e(c["id"])}" data-key="{k}">{k}. '
|
|
92
|
+
f'<b>{e(c["id"])}</b> <span class=mut>{e(c["desc"])}</span></button>')
|
|
93
|
+
parts.append('<div class=row><button name=a value=":skip" data-key="s">s skip</button>'
|
|
94
|
+
'<button name=a value=":undo" data-key="u">u undo</button>'
|
|
95
|
+
'<button name=a value=":quit" data-key="q">q done</button></div></form>')
|
|
96
|
+
parts.append(_progress(task, gold) + f"<script>{_JS}</script></body></html>")
|
|
97
|
+
return "".join(parts)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _teachers(silver: dict[str, Any] | None) -> list[tuple[str, str, float]]:
|
|
101
|
+
return [(t, top_label(d), max(d.values())) for t, d in sorted((silver or {}).items())
|
|
102
|
+
if d]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _drop_last(task: str, unit_id: str) -> None:
|
|
106
|
+
"""Undo: remove the newest gold row of the unit (the file is otherwise append-only)."""
|
|
107
|
+
p = eval_path(task, "gold")
|
|
108
|
+
rows = read_jsonl(p)
|
|
109
|
+
for i in range(len(rows) - 1, -1, -1):
|
|
110
|
+
if rows[i].get("unit_id") == unit_id:
|
|
111
|
+
del rows[i]
|
|
112
|
+
break
|
|
113
|
+
write_jsonl(p, rows)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class _Sitting:
|
|
117
|
+
"""One sitting of the gold page: the queue, the position in it, and what can be undone."""
|
|
118
|
+
|
|
119
|
+
def __init__(self, task: str, reqs: dict[str, Request], today: date | None) -> None:
|
|
120
|
+
self.task, self.silver, self.gold = task, silver_by_unit(task), gold_latest(task)
|
|
121
|
+
self.queue = stratified([r for u, r in reqs.items() if u not in self.gold],
|
|
122
|
+
lambda r: stratum(self.silver.get(r.unit_id)), seed=task)
|
|
123
|
+
self.pos = 0
|
|
124
|
+
self.hist: list[tuple[str, str]] = []
|
|
125
|
+
self.last: dict[str, Any] | None = None
|
|
126
|
+
self.counts: Counter[str] = Counter(answered=0, skipped=0)
|
|
127
|
+
self.ids = {c for c, _d in TASKS[task].candidates}
|
|
128
|
+
self.day = (today or date.today()).isoformat()
|
|
129
|
+
|
|
130
|
+
def _item(self) -> Request | None:
|
|
131
|
+
return self.queue[self.pos] if self.pos < len(self.queue) else None
|
|
132
|
+
|
|
133
|
+
def page(self, token: str) -> bytes:
|
|
134
|
+
return render(self.task, token, self._item(), self.gold, self.last,
|
|
135
|
+
len(self.queue) - self.pos, self.counts).encode()
|
|
136
|
+
|
|
137
|
+
def _undo(self) -> None:
|
|
138
|
+
kind, uid = self.hist.pop()
|
|
139
|
+
self.pos -= 1
|
|
140
|
+
self.last = None
|
|
141
|
+
if kind == "gold":
|
|
142
|
+
_drop_last(self.task, uid)
|
|
143
|
+
self.gold.pop(uid, None)
|
|
144
|
+
self.counts["answered"] -= 1
|
|
145
|
+
else:
|
|
146
|
+
self.counts["skipped"] -= 1
|
|
147
|
+
|
|
148
|
+
def _answer(self, u: str, a: str) -> None:
|
|
149
|
+
append_jsonl(eval_path(self.task, "gold"), {"unit_id": u, "gold": a, "labeller": LABELLER,
|
|
150
|
+
"date": self.day,
|
|
151
|
+
"task_ver": TASKS[self.task].ver})
|
|
152
|
+
self.gold[u] = a
|
|
153
|
+
self.hist.append(("gold", u))
|
|
154
|
+
self.pos += 1
|
|
155
|
+
self.last = {"gold": a, "teachers": _teachers(self.silver.get(u))}
|
|
156
|
+
self.counts["answered"] += 1
|
|
157
|
+
|
|
158
|
+
def act(self, form: dict[str, Any]) -> bool:
|
|
159
|
+
"""Apply one answer; True when the sitting ends."""
|
|
160
|
+
a, u = (form.get("a") or [""])[0], (form.get("u") or [""])[0]
|
|
161
|
+
item = self._item()
|
|
162
|
+
if a == ":quit":
|
|
163
|
+
return True
|
|
164
|
+
if a == ":undo" and self.hist:
|
|
165
|
+
self._undo()
|
|
166
|
+
elif item is None or u != item.unit_id:
|
|
167
|
+
pass # a stale page (another tab, a double press)
|
|
168
|
+
elif a == ":skip":
|
|
169
|
+
self.hist.append(("skip", u))
|
|
170
|
+
self.pos += 1
|
|
171
|
+
self.counts["skipped"] += 1
|
|
172
|
+
elif a in self.ids:
|
|
173
|
+
self._answer(u, a)
|
|
174
|
+
return False
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def serve(task: str, reqs: dict[str, Request], *, open_browser: bool = True,
|
|
178
|
+
on_ready: Callable[[str], None] | None = None, today: date | None = None) -> Counter[str]:
|
|
179
|
+
"""Serve the page until q (or Ctrl-C). Returns {answered, skipped} of this sitting."""
|
|
180
|
+
token = local_page.new_token()
|
|
181
|
+
sitting = _Sitting(task, reqs, today)
|
|
182
|
+
|
|
183
|
+
def on_post(form: dict[str, Any]) -> bytes | None:
|
|
184
|
+
if sitting.act(form):
|
|
185
|
+
return f"<p>Saved {sitting.counts['answered']} answers. You can close this tab.</p>".encode()
|
|
186
|
+
return None
|
|
187
|
+
|
|
188
|
+
local_page.serve(token, lambda tail: sitting.page(token) if tail == "" else None, on_post,
|
|
189
|
+
"gold page at {url} (q on the page, or Ctrl-C, to stop; answers are already "
|
|
190
|
+
"saved)", open_browser=open_browser, on_ready=on_ready)
|
|
191
|
+
return sitting.counts
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def run_gold_review(cfg: Config, task: str | None, days: int = LABEL_DAYS, *, src: Source | None = None,
|
|
195
|
+
open_browser: bool = True,
|
|
196
|
+
on_ready: Callable[[str], None] | None = None,
|
|
197
|
+
now: datetime | None = None) -> dict[str, Any]:
|
|
198
|
+
task = task or next((t for t in cfg.label_tasks if t in TASKS), "steer_intent")
|
|
199
|
+
today = (now or datetime.now(timezone.utc)).astimezone().date()
|
|
200
|
+
counts = serve(task, window_units(cfg, task, days, src, today), open_browser=open_browser,
|
|
201
|
+
on_ready=on_ready, today=today)
|
|
202
|
+
n = Counter(gold_latest(task).values())
|
|
203
|
+
return {**counts, "task": task, "gold": {c: n[c] for c, _d in TASKS[task].candidates}}
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def print_review(report: dict[str, Any]) -> None:
|
|
207
|
+
task = report["task"]
|
|
208
|
+
print(f"label review {task}: {report['answered']} answered, {report['skipped']} skipped; "
|
|
209
|
+
f"gold per label {json.dumps(report['gold'])} (floor {FLOORS[task][0]})")
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
"""Apply saved local decision recipes as estimates, separately from calibrated labels."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import re
|
|
9
|
+
import subprocess
|
|
10
|
+
import sys
|
|
11
|
+
import time
|
|
12
|
+
from datetime import datetime, timedelta, timezone
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, Callable
|
|
15
|
+
|
|
16
|
+
from ..config import AutError, Config, config_path
|
|
17
|
+
from ..event import ActivityEvent, make_event_id
|
|
18
|
+
from ..state import LockBusy, _locked, atomic_write_text, load_json_dict
|
|
19
|
+
from .backends import BackendUnavailable, find_model, make_backend
|
|
20
|
+
from .core import PROMPT_VER, Backend, Choice, Request, calibrate, data_dir, softmax, top_label
|
|
21
|
+
from .tasks import TASKS
|
|
22
|
+
|
|
23
|
+
POLICY = "local-decision-v1"
|
|
24
|
+
TASK_NAMES = {"steer_intent", "work_category", "soft_outcome", "writing_activity"}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def writing_policy_path() -> Path:
|
|
28
|
+
return config_path().with_name("label-local-writing.json")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def reserve_label_time(root: Path, now: datetime) -> str:
|
|
32
|
+
"""Reserve a strictly newer local label time across runs and replay jobs."""
|
|
33
|
+
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
34
|
+
path = root / "generation.json"
|
|
35
|
+
with _locked(path):
|
|
36
|
+
atomic_write_text(root / ".gitignore", "*\n")
|
|
37
|
+
previous = -1
|
|
38
|
+
if path.exists():
|
|
39
|
+
saved = load_json_dict(path).get("last_ms")
|
|
40
|
+
if type(saved) is not int or saved < 0:
|
|
41
|
+
raise AutError("label: invalid local generation state; it was not reset")
|
|
42
|
+
previous = saved
|
|
43
|
+
elif any(child.is_dir() for child in root.iterdir()):
|
|
44
|
+
raise AutError("label: missing generation state; it was not reset")
|
|
45
|
+
stamp = max(int(now.timestamp() * 1000), previous + 1)
|
|
46
|
+
try:
|
|
47
|
+
value = (
|
|
48
|
+
datetime(1970, 1, 1, tzinfo=timezone.utc) + timedelta(milliseconds=stamp)
|
|
49
|
+
).isoformat(timespec="milliseconds")
|
|
50
|
+
except OverflowError as exc:
|
|
51
|
+
raise AutError("label: invalid local generation state; it was not reset") from exc
|
|
52
|
+
atomic_write_text(path, json.dumps({"last_ms": stamp}))
|
|
53
|
+
return value
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _number(value: Any) -> bool:
|
|
57
|
+
try:
|
|
58
|
+
return type(value) in (int, float) and math.isfinite(value)
|
|
59
|
+
except OverflowError:
|
|
60
|
+
return False
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _require(condition: Any) -> None:
|
|
64
|
+
if not condition:
|
|
65
|
+
raise ValueError("invalid local decision policy")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def load_policy(path: Path | None = None) -> tuple[dict[str, Any], str]:
|
|
69
|
+
explicit = path is not None
|
|
70
|
+
path = path or config_path().with_name("label-local.json")
|
|
71
|
+
try:
|
|
72
|
+
obj = json.loads(path.read_text())
|
|
73
|
+
_require(obj["version"] == POLICY and obj["tasks"])
|
|
74
|
+
if "writing_activity" in obj["tasks"]:
|
|
75
|
+
_require(explicit and set(obj["tasks"]) == {"writing_activity"})
|
|
76
|
+
for task, row in obj["tasks"].items():
|
|
77
|
+
_require(task in TASK_NAMES)
|
|
78
|
+
_require(row["prompt_ver"] == f"{task}@{TASKS[task].ver}/p{PROMPT_VER}")
|
|
79
|
+
_require(_number(row["minimum_score"]) and 0.5 <= row["minimum_score"] <= 1)
|
|
80
|
+
_require(re.fullmatch(r"[a-f0-9]{64}", row["source_eval_sha256"]))
|
|
81
|
+
if "binary_decision_threshold" in row:
|
|
82
|
+
boundary = row["binary_decision_threshold"]
|
|
83
|
+
_require(TASKS[task].decision == "boolean" and len(TASKS[task].candidates) == 2)
|
|
84
|
+
_require(isinstance(boundary, dict) and set(boundary) == {"label", "value"})
|
|
85
|
+
_require(isinstance(boundary["label"], str)
|
|
86
|
+
and boundary["label"] in {c for c, _ in TASKS[task].candidates})
|
|
87
|
+
_require(_number(boundary["value"]) and 1e-6 < boundary["value"] < 1 - 1e-6)
|
|
88
|
+
recipe = row["recipe"]
|
|
89
|
+
_require(re.fullmatch(r"[a-zA-Z0-9._-]{1,100}", recipe["id"]))
|
|
90
|
+
_require(recipe["method"] in {"single", "majority", "stack"})
|
|
91
|
+
_require(len(recipe["members"]) in ({1} if recipe["method"] == "single" else {3, 5}))
|
|
92
|
+
_require(len({m["model_id"] for m in recipe["members"]}) == len(recipe["members"]))
|
|
93
|
+
for member in recipe["members"]:
|
|
94
|
+
model = find_model(member["model_id"])
|
|
95
|
+
if model is None:
|
|
96
|
+
raise ValueError("unknown local decision model")
|
|
97
|
+
_require(
|
|
98
|
+
model.backend
|
|
99
|
+
in {"mlx_pplx", "mlx_elephant", "mlx_clef", "mlx_card", "llama_server"}
|
|
100
|
+
)
|
|
101
|
+
_require(member["backend"] == model.backend)
|
|
102
|
+
_require(re.fullmatch(r"[A-Za-z0-9._-]{1,100}", member["model_rev"]))
|
|
103
|
+
_require(_number(member["temperature"]) and member["temperature"] > 0)
|
|
104
|
+
if recipe["method"] == "stack":
|
|
105
|
+
stack = recipe["stack"]
|
|
106
|
+
_require(
|
|
107
|
+
len(stack["a"]) == len(recipe["members"]) and all(map(_number, stack["a"]))
|
|
108
|
+
)
|
|
109
|
+
_require(set(stack["b"]) <= {c for c, _ in TASKS[task].candidates})
|
|
110
|
+
_require(all(map(_number, stack["b"].values())))
|
|
111
|
+
digest = hashlib.sha256(
|
|
112
|
+
json.dumps(obj, sort_keys=True, allow_nan=False).encode()
|
|
113
|
+
).hexdigest()
|
|
114
|
+
return obj, digest
|
|
115
|
+
except (AssertionError, KeyError, TypeError, ValueError, OSError, AttributeError) as exc:
|
|
116
|
+
raise AutError(
|
|
117
|
+
f"label: invalid or missing {path.name}; local estimates stay disabled"
|
|
118
|
+
) from exc
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def memory_guard() -> int | None:
|
|
122
|
+
"""Refuse a local model run below 15% free memory; return the free percentage (None off macOS)."""
|
|
123
|
+
if sys.platform != "darwin":
|
|
124
|
+
return None
|
|
125
|
+
try:
|
|
126
|
+
result = subprocess.run(
|
|
127
|
+
["/usr/bin/memory_pressure", "-Q"], capture_output=True, text=True, timeout=10
|
|
128
|
+
)
|
|
129
|
+
match = re.search(r"memory free percentage:\s*(\d+)%", result.stdout)
|
|
130
|
+
if result.returncode or not match or int(match[1]) < 15:
|
|
131
|
+
raise BackendUnavailable("label: local model run waits for at least 15% free memory")
|
|
132
|
+
return int(match[1])
|
|
133
|
+
except (OSError, subprocess.SubprocessError) as exc:
|
|
134
|
+
raise BackendUnavailable("label: could not check local model memory headroom") from exc
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def combine(
|
|
138
|
+
method: str, dists: list[dict[str, float]], stack: dict[str, Any] | None = None
|
|
139
|
+
) -> dict[str, float]:
|
|
140
|
+
if method == "single":
|
|
141
|
+
return dists[0]
|
|
142
|
+
labels = list(dists[0])
|
|
143
|
+
if method == "stack" and stack is not None:
|
|
144
|
+
logits = [
|
|
145
|
+
sum(a * math.log(max(d[label], 1e-12)) for a, d in zip(stack["a"], dists))
|
|
146
|
+
+ stack["b"].get(label, 0.0)
|
|
147
|
+
for label in labels
|
|
148
|
+
]
|
|
149
|
+
return dict(zip(labels, softmax(logits)))
|
|
150
|
+
return {label: sum(top_label(d) == label for d in dists) / len(dists) for label in labels}
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def score_member(
|
|
154
|
+
member: dict[str, Any],
|
|
155
|
+
requests: list[Request],
|
|
156
|
+
deadline: float,
|
|
157
|
+
factory: Callable[[dict[str, Any]], Backend],
|
|
158
|
+
stopped: Callable[[], bool] | None = None,
|
|
159
|
+
) -> list[dict[str, float]]:
|
|
160
|
+
def check() -> None:
|
|
161
|
+
if time.monotonic() >= deadline or (stopped is not None and stopped()):
|
|
162
|
+
raise BackendUnavailable("label: local model budget reached; unfinished batch retries")
|
|
163
|
+
memory_guard()
|
|
164
|
+
|
|
165
|
+
check()
|
|
166
|
+
backend = factory(member)
|
|
167
|
+
try:
|
|
168
|
+
check()
|
|
169
|
+
getattr(backend, "prepare", lambda: None)()
|
|
170
|
+
if backend.model_rev != member["model_rev"]:
|
|
171
|
+
raise BackendUnavailable("label: local model revision differs from saved recipe")
|
|
172
|
+
scores = []
|
|
173
|
+
for item in requests:
|
|
174
|
+
check()
|
|
175
|
+
raw = backend.distribution(item)
|
|
176
|
+
if (
|
|
177
|
+
set(raw) != {c["id"] for c in item.candidates}
|
|
178
|
+
or not all(_number(v) and v >= 0 for v in raw.values())
|
|
179
|
+
or sum(raw.values()) <= 0
|
|
180
|
+
):
|
|
181
|
+
raise BackendUnavailable("label: local model returned an invalid distribution")
|
|
182
|
+
scores.append(calibrate(raw, member["temperature"]))
|
|
183
|
+
return scores
|
|
184
|
+
finally:
|
|
185
|
+
getattr(backend, "close", lambda: None)()
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
class LocalBackend:
|
|
189
|
+
"""Apply the saved recipe, using serial scoring unless execution settings qualify."""
|
|
190
|
+
|
|
191
|
+
backend = "local_decision"
|
|
192
|
+
size_mb: float | None = None
|
|
193
|
+
|
|
194
|
+
def __init__(
|
|
195
|
+
self,
|
|
196
|
+
task: str,
|
|
197
|
+
row: dict[str, Any],
|
|
198
|
+
policy_id: str,
|
|
199
|
+
cfg: Config,
|
|
200
|
+
member_for: Callable[[dict[str, Any]], Backend] | None = None,
|
|
201
|
+
) -> None:
|
|
202
|
+
self.row, self.recipe, self.policy_id = row, row["recipe"], policy_id
|
|
203
|
+
self.model_id, self.model_rev = "local/" + self.recipe["id"], policy_id
|
|
204
|
+
self.member_for = member_for or (
|
|
205
|
+
lambda m: make_backend(
|
|
206
|
+
task,
|
|
207
|
+
Choice(backend=m["backend"], model_id=m["model_id"], model_rev=m["model_rev"]),
|
|
208
|
+
cfg.label_model,
|
|
209
|
+
cfg.label_llama_server,
|
|
210
|
+
)
|
|
211
|
+
)
|
|
212
|
+
self._requests: list[Request] = []
|
|
213
|
+
self._offset, self._deadline = 0, 0.0
|
|
214
|
+
self._scores: dict[str, dict[str, float]] = {}
|
|
215
|
+
from .local_parallel import load_profile
|
|
216
|
+
|
|
217
|
+
self.runtime = load_profile(task, row) if member_for is None else None
|
|
218
|
+
self.label_model, self.llama_server = cfg.label_model, cfg.label_llama_server
|
|
219
|
+
|
|
220
|
+
@property
|
|
221
|
+
def prefetch_size(self) -> int:
|
|
222
|
+
return self.runtime.group_size if self.runtime is not None else 50
|
|
223
|
+
|
|
224
|
+
def prepare_requests(self, requests: list[Request], deadline: float) -> None:
|
|
225
|
+
self._requests, self._deadline = requests, deadline
|
|
226
|
+
self._offset = 0
|
|
227
|
+
self._scores.clear()
|
|
228
|
+
|
|
229
|
+
def distribution(self, req: Request) -> dict[str, float]:
|
|
230
|
+
if req.unit_id not in self._scores:
|
|
231
|
+
try:
|
|
232
|
+
with _locked(data_dir() / "label" / "model-runtime.json", blocking=False):
|
|
233
|
+
self._score_next(req)
|
|
234
|
+
except LockBusy as exc:
|
|
235
|
+
raise BackendUnavailable("label: local model scoring is already running") from exc
|
|
236
|
+
return self._scores.pop(req.unit_id)
|
|
237
|
+
|
|
238
|
+
def _score_next(self, req: Request) -> None:
|
|
239
|
+
from .local_parallel import MemoryAdmissionError, score_parallel
|
|
240
|
+
|
|
241
|
+
batch = self._requests[self._offset : self._offset + self.prefetch_size]
|
|
242
|
+
if not batch or batch[0].unit_id != req.unit_id:
|
|
243
|
+
raise BackendUnavailable("label: local model request order changed")
|
|
244
|
+
parallel = False
|
|
245
|
+
if self.runtime is not None:
|
|
246
|
+
try:
|
|
247
|
+
scores = score_parallel(
|
|
248
|
+
batch,
|
|
249
|
+
self.row,
|
|
250
|
+
self.label_model,
|
|
251
|
+
self.llama_server,
|
|
252
|
+
self.runtime,
|
|
253
|
+
self._deadline,
|
|
254
|
+
)
|
|
255
|
+
parallel = True
|
|
256
|
+
except MemoryAdmissionError:
|
|
257
|
+
batch = batch[:50]
|
|
258
|
+
print(
|
|
259
|
+
"aut: local parallel scoring waits for memory; using serial groups",
|
|
260
|
+
file=sys.stderr,
|
|
261
|
+
)
|
|
262
|
+
if not parallel:
|
|
263
|
+
members = [
|
|
264
|
+
score_member(m, batch, self._deadline, self.member_for)
|
|
265
|
+
for m in self.recipe["members"]
|
|
266
|
+
]
|
|
267
|
+
scores = {
|
|
268
|
+
item.unit_id: combine(
|
|
269
|
+
self.recipe["method"], [m[i] for m in members], self.recipe.get("stack")
|
|
270
|
+
)
|
|
271
|
+
for i, item in enumerate(batch)
|
|
272
|
+
}
|
|
273
|
+
if set(scores) != {item.unit_id for item in batch}:
|
|
274
|
+
raise BackendUnavailable("label: saved local recipe returned invalid scores")
|
|
275
|
+
for item in batch:
|
|
276
|
+
dist = scores[item.unit_id]
|
|
277
|
+
if (
|
|
278
|
+
set(dist) != {c["id"] for c in item.candidates}
|
|
279
|
+
or not all(_number(v) and 0 <= v <= 1 for v in dist.values())
|
|
280
|
+
or abs(sum(dist.values()) - 1) > 1e-5
|
|
281
|
+
):
|
|
282
|
+
raise BackendUnavailable("label: saved local recipe returned invalid scores")
|
|
283
|
+
if boundary := self.row.get("binary_decision_threshold"):
|
|
284
|
+
threshold = boundary["value"]
|
|
285
|
+
scores[item.unit_id] = calibrate(dist, platt={
|
|
286
|
+
"yes": boundary["label"], "a": 1.,
|
|
287
|
+
"b": -math.log(threshold / (1 - threshold)),
|
|
288
|
+
})
|
|
289
|
+
self._scores = scores
|
|
290
|
+
self._offset += len(batch)
|
|
291
|
+
|
|
292
|
+
def close(self) -> None:
|
|
293
|
+
self._scores.clear()
|
|
294
|
+
self._requests.clear()
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def decorate(
|
|
298
|
+
event: ActivityEvent, req: Request, backend: LocalBackend, probability: float
|
|
299
|
+
) -> ActivityEvent:
|
|
300
|
+
attrs = event.attrs
|
|
301
|
+
uncertain = probability <= backend.row["minimum_score"]
|
|
302
|
+
attrs.update(
|
|
303
|
+
task="working.local." + req.task,
|
|
304
|
+
label_policy=POLICY,
|
|
305
|
+
policy_id=backend.policy_id,
|
|
306
|
+
provisional=True,
|
|
307
|
+
human_verified=False,
|
|
308
|
+
calibrated=False,
|
|
309
|
+
native_score=probability,
|
|
310
|
+
source_scope="ongoing_local_eligible",
|
|
311
|
+
population_weight=1,
|
|
312
|
+
label_source="local_decision",
|
|
313
|
+
predicted_label=attrs["label"],
|
|
314
|
+
below_tau=uncertain,
|
|
315
|
+
source_eval_sha256=backend.row["source_eval_sha256"],
|
|
316
|
+
input_hash=hashlib.sha256(json.dumps(req.facts, sort_keys=True).encode()).hexdigest(),
|
|
317
|
+
model_members=[
|
|
318
|
+
{k: m[k] for k in ("model_id", "model_rev")} for m in backend.recipe["members"]
|
|
319
|
+
],
|
|
320
|
+
)
|
|
321
|
+
if uncertain:
|
|
322
|
+
attrs["label"] = "unknown"
|
|
323
|
+
event.event_id = make_event_id(
|
|
324
|
+
"local-estimate", event.event_id, backend.policy_id, req.unit, attrs["input_hash"]
|
|
325
|
+
)
|
|
326
|
+
return event
|