agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
"""`aut label run [--task T] [--since D] [--dry-run]` and `aut label backfill --task T --days N`.
|
|
2
|
+
|
|
3
|
+
Nightly after the scan (its own step in aut-scan.sh, its own heartbeat collector `label`). Per
|
|
4
|
+
task: build the units since the watermark, score them inside the time budget, send the label rows
|
|
5
|
+
in batches and advance the watermark after each delivered batch, so an interrupted run resumes
|
|
6
|
+
where it stopped (a rerun of a unit is a no-op at the sink). The run never logs text.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import sys
|
|
13
|
+
import time
|
|
14
|
+
from collections import Counter, defaultdict
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from datetime import date, datetime, timedelta, timezone
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any, Callable
|
|
19
|
+
|
|
20
|
+
from ..config import LABEL_NOTE, AutError, Config
|
|
21
|
+
from ..episode_store import Episode
|
|
22
|
+
from ..event import ActivityEvent
|
|
23
|
+
from ..push import job_heartbeat, send_activity_rows
|
|
24
|
+
from ..repo_keys import refuse_new_salt
|
|
25
|
+
from ..sink import connect
|
|
26
|
+
from ..state import atomic_write_text, collector_wm, load_json_dict, load_state, save_state
|
|
27
|
+
from .backends import BackendUnavailable, make_backend
|
|
28
|
+
from .core import (Backend, Choice, NoteBackend, current_labels, data_dir, decide, label_event,
|
|
29
|
+
load_choice, local_days, local_midnight)
|
|
30
|
+
from .tasks import OUTCOME_DAYS, TASKS, Source, SourceUnavailable, Task, units
|
|
31
|
+
from .local import LocalBackend, decorate, load_policy, reserve_label_time, writing_policy_path
|
|
32
|
+
from .covered_run import covered_outcomes
|
|
33
|
+
|
|
34
|
+
BATCH = 50
|
|
35
|
+
# The day tasks built from agents.episode, which the sink rebuilds once a night: they wait for it.
|
|
36
|
+
_EPISODE_TASKS = frozenset({"work_category", "writing_activity", LABEL_NOTE})
|
|
37
|
+
# One task's counts, read as 0 when unset, plus `labels`: the rows per label.
|
|
38
|
+
TaskReport = defaultdict[str, Any]
|
|
39
|
+
NAME = "label"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _cache_path() -> Path:
|
|
43
|
+
return data_dir() / "label" / "cache.json"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True, slots=True)
|
|
47
|
+
class _Run:
|
|
48
|
+
"""What stays fixed across the tasks of one label run. wm is state's label section."""
|
|
49
|
+
cfg: Config
|
|
50
|
+
dsn: str
|
|
51
|
+
src: Source
|
|
52
|
+
dry_run: bool
|
|
53
|
+
deadline: float
|
|
54
|
+
wm: dict[str, Any]
|
|
55
|
+
state: dict[str, Any]
|
|
56
|
+
cache: dict[str, Any]
|
|
57
|
+
labelled_at: str
|
|
58
|
+
ready: datetime | None = None # Source.episodes_ready, read once per run
|
|
59
|
+
state_name: str = NAME
|
|
60
|
+
local_estimates: bool = False
|
|
61
|
+
|
|
62
|
+
def advance(self, name: str, ts: str | None) -> None:
|
|
63
|
+
"""Save a task's watermark, never backwards: a backfill over old days leaves the nightly
|
|
64
|
+
watermark where it was."""
|
|
65
|
+
if ts and (not self.wm.get(name)
|
|
66
|
+
or datetime.fromisoformat(ts) > datetime.fromisoformat(self.wm[name])):
|
|
67
|
+
self.wm[name] = ts
|
|
68
|
+
save_state(self.state, [self.state_name])
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _window(name: str, wm: dict[str, Any], since: datetime | None, now: datetime,
|
|
72
|
+
ready: datetime | None = None, *, mature_outcomes: bool = False,
|
|
73
|
+
) -> tuple[datetime, datetime | None]:
|
|
74
|
+
"""(since, until) of one task. until None: up to now, and the watermark becomes the newest
|
|
75
|
+
unit's ts. A day task covers whole past days (see _day_window)."""
|
|
76
|
+
window = TASKS[name].window
|
|
77
|
+
if window == "recent":
|
|
78
|
+
if since:
|
|
79
|
+
scope = "recent and pending mature outcomes" if mature_outcomes else \
|
|
80
|
+
f"the last {OUTCOME_DAYS} days"
|
|
81
|
+
print(f"aut: label {name}: --since and --days do not apply; it always covers "
|
|
82
|
+
f"{scope}", file=sys.stderr)
|
|
83
|
+
return now - timedelta(days=OUTCOME_DAYS), None
|
|
84
|
+
if window == "day":
|
|
85
|
+
return _day_window(name, wm, since, now, ready)
|
|
86
|
+
return _start(name, wm, since, now), None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _start(name: str, wm: dict[str, Any], since: datetime | None, now: datetime) -> datetime:
|
|
90
|
+
"""Where a task picks up: `since`, else its watermark, else a day ago."""
|
|
91
|
+
if since:
|
|
92
|
+
return since
|
|
93
|
+
if wm.get(name):
|
|
94
|
+
return datetime.fromisoformat(wm[name])
|
|
95
|
+
return local_midnight(now.astimezone().date()) - timedelta(1)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _day_window(name: str, wm: dict[str, Any], since: datetime | None, now: datetime,
|
|
99
|
+
ready: datetime | None = None) -> tuple[datetime, datetime]:
|
|
100
|
+
"""(since, until) over whole past days: until is today's local midnight, or for a task built
|
|
101
|
+
from episodes the midnight that starts the day of `ready`, so a day whose episodes are not
|
|
102
|
+
rebuilt yet (a month delivered after a gap) waits for the next rebuild instead of being passed
|
|
103
|
+
for good. Never before since: then the window is empty."""
|
|
104
|
+
start = local_midnight(_start(name, wm, since, now).astimezone().date())
|
|
105
|
+
until = local_midnight(now.astimezone().date())
|
|
106
|
+
if name in _EPISODE_TASKS:
|
|
107
|
+
until = start if ready is None else max(
|
|
108
|
+
start, min(until, local_midnight(ready.astimezone().date())))
|
|
109
|
+
return start, until
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _note_prompt(day: date, labels: list[tuple[Any, ...]], episodes: list[Episode]) -> str:
|
|
113
|
+
"""Counts only: the day's accepted labels and episode facts, never a steer text or title."""
|
|
114
|
+
by = Counter((a["task"], a["label"]) for a in current_labels(labels) if not a["below_tau"])
|
|
115
|
+
lines = [f"Write a short private note (3 to 5 sentences) about the work day {day}. Use only "
|
|
116
|
+
"these facts and invent nothing.", "", "Confident labels (task: label count):"]
|
|
117
|
+
lines += [f"- {t}: {lab} {n}" for (t, lab), n in sorted(by.items())] or ["- none"]
|
|
118
|
+
outs = Counter(e["outcome"] for e in episodes)
|
|
119
|
+
lines += ["", f"Episodes active: {len(episodes)} (outcomes: "
|
|
120
|
+
f"{', '.join(f'{k} {v}' for k, v in sorted(outs.items())) or 'none'}), commits "
|
|
121
|
+
f"{sum(e['n_commits'] for e in episodes)}, sessions "
|
|
122
|
+
f"{sum(e['n_sessions'] for e in episodes)}."]
|
|
123
|
+
return "\n".join(lines)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _insert_note(dsn: str, day: date, text: str, model_id: str) -> None:
|
|
127
|
+
with connect(dsn) as conn:
|
|
128
|
+
conn.execute("INSERT INTO agents.daily_note (day, text, model_id, created_at) "
|
|
129
|
+
"VALUES (%s, %s, %s, now()) ON CONFLICT (day) DO NOTHING",
|
|
130
|
+
(day, text, model_id))
|
|
131
|
+
conn.commit()
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _run_notes(run: _Run, backend: NoteBackend, since: datetime, until: datetime) -> TaskReport:
|
|
135
|
+
c: TaskReport = defaultdict(int)
|
|
136
|
+
for d in local_days(since, until):
|
|
137
|
+
if time.monotonic() >= run.deadline:
|
|
138
|
+
c["cut"] = 1
|
|
139
|
+
break
|
|
140
|
+
start, end = local_midnight(d), local_midnight(d + timedelta(1))
|
|
141
|
+
eps = [e for e in run.src.episodes(start) if e["first_ts"] < end]
|
|
142
|
+
text = backend.generate(_note_prompt(d, run.src.labels(start, end), eps))
|
|
143
|
+
c["notes"] += 1
|
|
144
|
+
if not run.dry_run:
|
|
145
|
+
_insert_note(run.dsn, d, text, backend.model_id)
|
|
146
|
+
run.advance(LABEL_NOTE, end.isoformat())
|
|
147
|
+
return c
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _flush(run: _Run, task: Task, key: str, pending: list[ActivityEvent],
|
|
151
|
+
next_wm: str | None) -> None:
|
|
152
|
+
"""Deliver a batch, then remember it and move the watermark. A dry run only drops it."""
|
|
153
|
+
if not run.dry_run:
|
|
154
|
+
if pending:
|
|
155
|
+
send_activity_rows(run.dsn, pending) # raises: watermark stays
|
|
156
|
+
if task.cached and pending:
|
|
157
|
+
run.cache.setdefault(key, []).extend(e.attrs["unit_id"] for e in pending)
|
|
158
|
+
atomic_write_text(_cache_path(), json.dumps(run.cache))
|
|
159
|
+
if task.window != "recent":
|
|
160
|
+
run.advance(task.name, next_wm)
|
|
161
|
+
pending.clear()
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _run_task(run: _Run, task: Task, choice: Choice, backend: Backend, since: datetime,
|
|
165
|
+
until: datetime | None) -> TaskReport:
|
|
166
|
+
c: TaskReport = defaultdict(int)
|
|
167
|
+
labels: Counter[str] = Counter()
|
|
168
|
+
c["labels"] = labels
|
|
169
|
+
key = f"{task.name}@{task.ver}|{backend.model_id}|{choice.calib_ver}"
|
|
170
|
+
done = set(run.cache.get(key, ())) if task.cached else set()
|
|
171
|
+
skipped = run.src.skipped["no_text"]
|
|
172
|
+
incomplete = len(run.src.incomplete_outcome_units)
|
|
173
|
+
reqs = [r for r in units(task.name, run.src, since, until) if r.unit_id not in done]
|
|
174
|
+
c["skipped_incomplete_outcome"] = len(run.src.incomplete_outcome_units) - incomplete
|
|
175
|
+
if task.name == "soft_outcome" and run.local_estimates and run.cfg.label_mature_outcomes:
|
|
176
|
+
if not isinstance(backend, LocalBackend):
|
|
177
|
+
raise BackendUnavailable("label: mature outcomes require a saved local recipe")
|
|
178
|
+
if not run.src.mature_outcomes:
|
|
179
|
+
raise BackendUnavailable("label: mature outcomes require an opted-in local source")
|
|
180
|
+
covered_outcomes(run, reqs, backend, choice, c, BATCH, send_activity_rows)
|
|
181
|
+
return c
|
|
182
|
+
if run.local_estimates:
|
|
183
|
+
if not isinstance(backend, LocalBackend):
|
|
184
|
+
raise BackendUnavailable("label: local estimates require a saved local decision recipe")
|
|
185
|
+
backend.prepare_requests(reqs, run.deadline)
|
|
186
|
+
c["provisional"] = 1
|
|
187
|
+
c["skipped_no_text"] = run.src.skipped["no_text"] - skipped
|
|
188
|
+
pending: list[ActivityEvent] = []
|
|
189
|
+
cut = None
|
|
190
|
+
for r in reqs:
|
|
191
|
+
if time.monotonic() >= run.deadline:
|
|
192
|
+
cut = r
|
|
193
|
+
break
|
|
194
|
+
raw = backend.distribution(r) # first: it can load the model, which sets model_rev
|
|
195
|
+
res = decide(r, raw, choice, backend.model_id, backend.model_rev)
|
|
196
|
+
ev = label_event(r, res, run.labelled_at)
|
|
197
|
+
if isinstance(backend, LocalBackend):
|
|
198
|
+
ev = decorate(ev, r, backend, res.prob)
|
|
199
|
+
ev.machine = run.cfg.machine
|
|
200
|
+
pending.append(ev)
|
|
201
|
+
c["rows"] += 1
|
|
202
|
+
c["below_tau"] += ev.attrs["below_tau"]
|
|
203
|
+
labels[ev.attrs["label"]] += 1
|
|
204
|
+
if len(pending) >= BATCH:
|
|
205
|
+
_flush(run, task, key, pending, r.ts)
|
|
206
|
+
if cut is not None:
|
|
207
|
+
c["cut"] = 1
|
|
208
|
+
_flush(run, task, key, pending, cut.ts)
|
|
209
|
+
else:
|
|
210
|
+
_flush(run, task, key, pending,
|
|
211
|
+
until.isoformat() if until else (reqs[-1].ts if reqs else None))
|
|
212
|
+
return c
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _label_one(run: _Run, name: str, since: datetime | None, now: datetime, choice: Choice,
|
|
216
|
+
backend_for: Callable[[str, Choice], Backend]) -> tuple[TaskReport, Exception | None]:
|
|
217
|
+
"""One task's report, and the error that stopped only it: its watermark stays, the others go
|
|
218
|
+
on, and the heartbeat row records it."""
|
|
219
|
+
try:
|
|
220
|
+
if name == "writing_activity" and not run.local_estimates:
|
|
221
|
+
raise BackendUnavailable("label: writing_activity requires --local-estimates")
|
|
222
|
+
if name != LABEL_NOTE:
|
|
223
|
+
start, until = _window(name, run.wm, since, now, run.ready,
|
|
224
|
+
mature_outcomes=run.local_estimates and run.cfg.label_mature_outcomes)
|
|
225
|
+
if TASKS[name].window == "recent":
|
|
226
|
+
until = now # All recent and mature requests in this run share one as-of time.
|
|
227
|
+
report = _run_task(run, TASKS[name], choice, backend_for(name, choice), start, until)
|
|
228
|
+
else:
|
|
229
|
+
start, until = _day_window(name, run.wm, since, now, run.ready)
|
|
230
|
+
backend = backend_for(name, choice)
|
|
231
|
+
if not isinstance(backend, NoteBackend): # label.json can name the encoder for it
|
|
232
|
+
raise BackendUnavailable(f"label: {name} needs a backend that can write text")
|
|
233
|
+
report = _run_notes(run, backend, start, until)
|
|
234
|
+
if name in _EPISODE_TASKS and until is not None \
|
|
235
|
+
and until < local_midnight(now.astimezone().date()):
|
|
236
|
+
report["unavailable"] = 1
|
|
237
|
+
stopped = SourceUnavailable("waiting for the episode rebuild")
|
|
238
|
+
print(f"aut: label {name}: {stopped}; saved progress will resume", file=sys.stderr)
|
|
239
|
+
return report, stopped
|
|
240
|
+
if report["cut"]:
|
|
241
|
+
return report, TimeoutError("label time budget reached")
|
|
242
|
+
return report, None
|
|
243
|
+
except SourceUnavailable as exc:
|
|
244
|
+
print(f"aut: label {name}: {exc}; retried on the next run", file=sys.stderr)
|
|
245
|
+
return defaultdict(int, unavailable=1), exc
|
|
246
|
+
except BackendUnavailable as exc: # the same, and the exit status says so
|
|
247
|
+
print(f"aut: {exc}", file=sys.stderr)
|
|
248
|
+
return defaultdict(int, failed=1), exc
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def run_label(cfg: Config, tasks: list[str] | None = None, since: datetime | None = None, *,
|
|
252
|
+
dry_run: bool = False, src: Source | None = None,
|
|
253
|
+
backend_for: Callable[[str, Choice], Backend] | None = None,
|
|
254
|
+
now: datetime | None = None, local_estimates: bool = False) -> dict[str, TaskReport]:
|
|
255
|
+
"""Label each task (default: config label_tasks). backend_for(task, choice) -> backend
|
|
256
|
+
(tests pass a fake). A dry run scores and counts, and writes nothing: no rows, watermark,
|
|
257
|
+
cache or heartbeat. A task whose backend cannot run reports failed = 1 and the others go on."""
|
|
258
|
+
dsn = cfg.require_pg("label (it reads the steer and episode rows)")
|
|
259
|
+
refuse_new_salt(cfg.repo_key_salt, dsn, cfg.machine)
|
|
260
|
+
started = datetime.now(timezone.utc)
|
|
261
|
+
labelled_at = started.isoformat(timespec="milliseconds")
|
|
262
|
+
now = now or started
|
|
263
|
+
src = src or Source(cfg, local_context=local_estimates)
|
|
264
|
+
loaded: dict[tuple[str | None, str | None, str | None, str | None], Backend] = {}
|
|
265
|
+
names = tasks if tasks is not None else cfg.label_tasks
|
|
266
|
+
policy, policy_id = load_policy() if local_estimates and set(names) != {"writing_activity"} else ({}, "")
|
|
267
|
+
if local_estimates and not dry_run:
|
|
268
|
+
labelled_at = reserve_label_time(data_dir() / "label" / "replay", started)
|
|
269
|
+
|
|
270
|
+
def default_backend(task: str, choice: Choice) -> Backend:
|
|
271
|
+
if local_estimates:
|
|
272
|
+
task_policy, task_policy_id = policy, policy_id
|
|
273
|
+
if task == "writing_activity":
|
|
274
|
+
try:
|
|
275
|
+
task_policy, task_policy_id = load_policy(writing_policy_path())
|
|
276
|
+
except AutError as exc:
|
|
277
|
+
raise BackendUnavailable(str(exc)) from exc
|
|
278
|
+
key = (task, "local", task_policy_id, None)
|
|
279
|
+
if key not in loaded:
|
|
280
|
+
if task not in task_policy["tasks"]:
|
|
281
|
+
raise BackendUnavailable("label: task has no saved local decision recipe")
|
|
282
|
+
loaded[key] = LocalBackend(task, task_policy["tasks"][task], task_policy_id, cfg)
|
|
283
|
+
return loaded[key]
|
|
284
|
+
# one model load per run, shared across tasks; an encoder is trained for one task alone
|
|
285
|
+
k = (task if choice.backend == "encoder" else None, choice.backend,
|
|
286
|
+
choice.model_id or cfg.label_model, choice.model_rev)
|
|
287
|
+
if k not in loaded:
|
|
288
|
+
loaded[k] = make_backend(task, choice, cfg.label_model, cfg.label_llama_server)
|
|
289
|
+
return loaded[k]
|
|
290
|
+
|
|
291
|
+
backend_for = backend_for or default_backend
|
|
292
|
+
state = load_state()
|
|
293
|
+
state_name = "label_local" if local_estimates else NAME
|
|
294
|
+
wm = collector_wm(state, state_name)
|
|
295
|
+
run = _Run(cfg=cfg, dsn=dsn, src=src, dry_run=dry_run,
|
|
296
|
+
deadline=time.monotonic() + cfg.label_budget_s, wm=wm, state=state,
|
|
297
|
+
cache=load_json_dict(_cache_path()), labelled_at=labelled_at,
|
|
298
|
+
ready=src.episodes_ready() if _EPISODE_TASKS & set(names) else None,
|
|
299
|
+
state_name=state_name, local_estimates=local_estimates)
|
|
300
|
+
report: dict[str, TaskReport] = {}
|
|
301
|
+
with job_heartbeat(cfg, state_name, started, dry_run=dry_run) as beat:
|
|
302
|
+
try:
|
|
303
|
+
for name in names:
|
|
304
|
+
choice = Choice() if local_estimates else load_choice(
|
|
305
|
+
name, TASKS[name].ver if name in TASKS else None)
|
|
306
|
+
report[name], stopped = _label_one(run, name, since, now, choice,
|
|
307
|
+
backend_for)
|
|
308
|
+
beat.events += report[name]["rows"] + report[name]["notes"]
|
|
309
|
+
if stopped:
|
|
310
|
+
beat.fail(stopped)
|
|
311
|
+
finally:
|
|
312
|
+
for b in loaded.values(): # a llama-server stops with the run
|
|
313
|
+
getattr(b, "close", lambda: None)()
|
|
314
|
+
return report
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def print_report(report: dict[str, TaskReport], *, dry_run: bool) -> None:
|
|
318
|
+
mode = "DRY-RUN (nothing written)" if dry_run else "written"
|
|
319
|
+
for name, c in report.items():
|
|
320
|
+
if c["failed"]: # its reason is already on stderr
|
|
321
|
+
continue
|
|
322
|
+
labels = dict(sorted(c.get("labels", {}).items()))
|
|
323
|
+
what = f"{c['notes']} notes" if name == LABEL_NOTE else \
|
|
324
|
+
f"{c['rows']} rows ({c['below_tau']} below tau), by label {json.dumps(labels)}"
|
|
325
|
+
if c.get("provisional"):
|
|
326
|
+
what = f"{c['rows']} provisional rows ({c['below_tau']} unknown), by label {json.dumps(labels)}"
|
|
327
|
+
print(f"label {name} {mode}: {what}")
|
|
328
|
+
if c["skipped_no_text"]: # a new laptop, or transcripts deleted: say so, once a run
|
|
329
|
+
print(f"aut: label {name}: {c['skipped_no_text']} steers skipped: their text is not on "
|
|
330
|
+
"this machine", file=sys.stderr)
|
|
331
|
+
if c["skipped_incomplete_outcome"]:
|
|
332
|
+
print(f"aut: label {name}: {c['skipped_incomplete_outcome']} outcomes unclassified: "
|
|
333
|
+
"incomplete prompts or no attributable final reply", file=sys.stderr)
|
|
334
|
+
if c["skipped_stale_outcome"]:
|
|
335
|
+
print(f"aut: label {name}: {c['skipped_stale_outcome']} changed or no longer no-code "
|
|
336
|
+
"outcomes skipped; current inputs wait for the next run", file=sys.stderr)
|
|
337
|
+
if c["cut"]:
|
|
338
|
+
print(f"aut: label {name}: stopped by the time budget (label_budget_s); the rest "
|
|
339
|
+
"waits for the next run", file=sys.stderr)
|