agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,209 @@
1
+ """`aut label review [--task T]`: label gold answers one unit at a time on a local page.
2
+
3
+ Served from 127.0.0.1 on a random port under a random token, like `aut episodes review`. Each
4
+ item shows its local, redacted text and the candidates; keys 1-9 answer, s skips, u undoes, q
5
+ ends. The teachers' (silver) answers show only after the item is answered. Items come stratified
6
+ by silver label (rare labels and teacher disagreement come up as often as common ones), and the
7
+ page shows gold per label against the floor and target. Answers append to <task>.jsonl at once
8
+ (unit_id, gold, labeller, date, task_ver; never text), so a sitting can stop any time and the
9
+ next one resumes with what is left.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import html
15
+ import json
16
+ from collections import Counter
17
+ from datetime import date, datetime, timezone
18
+ from typing import Any, Callable
19
+
20
+ from .. import local_page
21
+ from ..config import LABEL_DAYS, Config
22
+ from .core import Request, top_label
23
+ from .evalset import (FLOORS, append_jsonl, eval_path, gold_latest, read_jsonl,
24
+ silver_by_unit, stratified, stratum, window_units, write_jsonl)
25
+ from .tasks import TASKS, Source
26
+
27
+ LABELLER = "human"
28
+ _KEYS = "123456789"
29
+ _CSS = """pre { white-space: pre-wrap; word-break: break-word; background: transparent;
30
+ border: 1px solid var(--line); border-radius: 6px; padding: .5rem; margin: .2rem 0 .6rem; }
31
+ .mut { color: var(--mut); font-size: 13px; } dt { font-weight: 600; }
32
+ button { font: inherit; display: block; width: 100%; text-align: left; margin: .25rem 0;
33
+ padding: .45rem .6rem; border: 1px solid var(--line); border-radius: 6px;
34
+ background: transparent; color: var(--fg); cursor: pointer; }
35
+ button:focus-visible { outline: 2px solid var(--acc); }
36
+ .row button { display: inline-block; width: auto; margin-right: .4rem; }
37
+ table { border-collapse: collapse; font-size: 13px; } td { padding: 0 .6rem 0 0; }
38
+ .last { border-left: 3px solid var(--acc); padding-left: .6rem; }"""
39
+ _JS = """document.addEventListener('keydown', e => {
40
+ if (e.metaKey || e.ctrlKey || e.altKey) return;
41
+ const b = document.querySelector('[data-key="' + e.key.toLowerCase() + '"]');
42
+ if (b) { e.preventDefault(); b.click(); } });"""
43
+
44
+
45
+ def _facts(facts: dict[str, Any]) -> str:
46
+ e = html.escape
47
+ parts = ["<dl>"]
48
+ for k, v in facts.items():
49
+ if isinstance(v, str) and v:
50
+ parts.append(f"<dt>{e(k)}</dt><dd><pre>{e(v)}</pre></dd>")
51
+ elif isinstance(v, list) and v and all(isinstance(x, str) for x in v):
52
+ parts.append(f"<dt>{e(k)}</dt><dd>" + "".join(f"<pre>{e(x)}</pre>" for x in v)
53
+ + "</dd>")
54
+ elif v not in (None, "", [], {}):
55
+ parts.append(f"<dt>{e(k)}</dt><dd>{e(json.dumps(v, ensure_ascii=False))}</dd>")
56
+ return "".join(parts) + "</dl>"
57
+
58
+
59
+ def _progress(task: str, gold: dict[str, str]) -> str:
60
+ floor, target = FLOORS[task]
61
+ n = Counter(gold.values())
62
+ rows = "".join(f"<tr><td>{html.escape(c)}</td><td>{n[c]} / {floor}</td>"
63
+ f"<td class=mut>{'floor reached' if n[c] >= floor else ''}</td></tr>"
64
+ for c, _d in TASKS[task].candidates)
65
+ return (f"<details><summary>Gold per label (floor {floor}, target {target})</summary>"
66
+ f"<table>{rows}</table></details>")
67
+
68
+
69
+ def render(task: str, token: str, item: Request | None, gold: dict[str, Any], last: dict[str, Any] | None,
70
+ left: int, counts: Counter[str]) -> str:
71
+ e = html.escape
72
+ head = (local_page.head("Gold labels", "52rem", _CSS) + f"<h1>{e(task)}</h1>"
73
+ "<p class=mut>Private page on this machine only. Only your answer (a label and the "
74
+ f"unit id) is saved, on this machine. This session: {counts['answered']} answered, "
75
+ f"{counts['skipped']} skipped; {left} left.</p>")
76
+ parts = [head]
77
+ if last:
78
+ said = ", ".join(f"{e(t)}: {e(lab)} ({p:.0%})" for t, lab, p in last["teachers"]) \
79
+ or "no teacher label"
80
+ parts.append(f"<p class=last>Last item: you said <b>{e(last['gold'])}</b>; teachers "
81
+ f"said {said}.</p>")
82
+ parts.append(f'<form method="post" action="/{e(token)}">')
83
+ if item is None:
84
+ parts.append("<p>Nothing left to label in this window.</p>")
85
+ else:
86
+ parts.append(f'<input type=hidden name=u value="{e(item.unit_id)}">'
87
+ f"<p><b>{e(item.question)}</b> <span class=mut>({e(item.unit)})</span></p>"
88
+ + _facts(item.facts))
89
+ for i, c in enumerate(item.candidates):
90
+ k = _KEYS[i] if i < len(_KEYS) else ""
91
+ parts.append(f'<button name=a value="{e(c["id"])}" data-key="{k}">{k}. '
92
+ f'<b>{e(c["id"])}</b> <span class=mut>{e(c["desc"])}</span></button>')
93
+ parts.append('<div class=row><button name=a value=":skip" data-key="s">s skip</button>'
94
+ '<button name=a value=":undo" data-key="u">u undo</button>'
95
+ '<button name=a value=":quit" data-key="q">q done</button></div></form>')
96
+ parts.append(_progress(task, gold) + f"<script>{_JS}</script></body></html>")
97
+ return "".join(parts)
98
+
99
+
100
+ def _teachers(silver: dict[str, Any] | None) -> list[tuple[str, str, float]]:
101
+ return [(t, top_label(d), max(d.values())) for t, d in sorted((silver or {}).items())
102
+ if d]
103
+
104
+
105
+ def _drop_last(task: str, unit_id: str) -> None:
106
+ """Undo: remove the newest gold row of the unit (the file is otherwise append-only)."""
107
+ p = eval_path(task, "gold")
108
+ rows = read_jsonl(p)
109
+ for i in range(len(rows) - 1, -1, -1):
110
+ if rows[i].get("unit_id") == unit_id:
111
+ del rows[i]
112
+ break
113
+ write_jsonl(p, rows)
114
+
115
+
116
+ class _Sitting:
117
+ """One sitting of the gold page: the queue, the position in it, and what can be undone."""
118
+
119
+ def __init__(self, task: str, reqs: dict[str, Request], today: date | None) -> None:
120
+ self.task, self.silver, self.gold = task, silver_by_unit(task), gold_latest(task)
121
+ self.queue = stratified([r for u, r in reqs.items() if u not in self.gold],
122
+ lambda r: stratum(self.silver.get(r.unit_id)), seed=task)
123
+ self.pos = 0
124
+ self.hist: list[tuple[str, str]] = []
125
+ self.last: dict[str, Any] | None = None
126
+ self.counts: Counter[str] = Counter(answered=0, skipped=0)
127
+ self.ids = {c for c, _d in TASKS[task].candidates}
128
+ self.day = (today or date.today()).isoformat()
129
+
130
+ def _item(self) -> Request | None:
131
+ return self.queue[self.pos] if self.pos < len(self.queue) else None
132
+
133
+ def page(self, token: str) -> bytes:
134
+ return render(self.task, token, self._item(), self.gold, self.last,
135
+ len(self.queue) - self.pos, self.counts).encode()
136
+
137
+ def _undo(self) -> None:
138
+ kind, uid = self.hist.pop()
139
+ self.pos -= 1
140
+ self.last = None
141
+ if kind == "gold":
142
+ _drop_last(self.task, uid)
143
+ self.gold.pop(uid, None)
144
+ self.counts["answered"] -= 1
145
+ else:
146
+ self.counts["skipped"] -= 1
147
+
148
+ def _answer(self, u: str, a: str) -> None:
149
+ append_jsonl(eval_path(self.task, "gold"), {"unit_id": u, "gold": a, "labeller": LABELLER,
150
+ "date": self.day,
151
+ "task_ver": TASKS[self.task].ver})
152
+ self.gold[u] = a
153
+ self.hist.append(("gold", u))
154
+ self.pos += 1
155
+ self.last = {"gold": a, "teachers": _teachers(self.silver.get(u))}
156
+ self.counts["answered"] += 1
157
+
158
+ def act(self, form: dict[str, Any]) -> bool:
159
+ """Apply one answer; True when the sitting ends."""
160
+ a, u = (form.get("a") or [""])[0], (form.get("u") or [""])[0]
161
+ item = self._item()
162
+ if a == ":quit":
163
+ return True
164
+ if a == ":undo" and self.hist:
165
+ self._undo()
166
+ elif item is None or u != item.unit_id:
167
+ pass # a stale page (another tab, a double press)
168
+ elif a == ":skip":
169
+ self.hist.append(("skip", u))
170
+ self.pos += 1
171
+ self.counts["skipped"] += 1
172
+ elif a in self.ids:
173
+ self._answer(u, a)
174
+ return False
175
+
176
+
177
+ def serve(task: str, reqs: dict[str, Request], *, open_browser: bool = True,
178
+ on_ready: Callable[[str], None] | None = None, today: date | None = None) -> Counter[str]:
179
+ """Serve the page until q (or Ctrl-C). Returns {answered, skipped} of this sitting."""
180
+ token = local_page.new_token()
181
+ sitting = _Sitting(task, reqs, today)
182
+
183
+ def on_post(form: dict[str, Any]) -> bytes | None:
184
+ if sitting.act(form):
185
+ return f"<p>Saved {sitting.counts['answered']} answers. You can close this tab.</p>".encode()
186
+ return None
187
+
188
+ local_page.serve(token, lambda tail: sitting.page(token) if tail == "" else None, on_post,
189
+ "gold page at {url} (q on the page, or Ctrl-C, to stop; answers are already "
190
+ "saved)", open_browser=open_browser, on_ready=on_ready)
191
+ return sitting.counts
192
+
193
+
194
+ def run_gold_review(cfg: Config, task: str | None, days: int = LABEL_DAYS, *, src: Source | None = None,
195
+ open_browser: bool = True,
196
+ on_ready: Callable[[str], None] | None = None,
197
+ now: datetime | None = None) -> dict[str, Any]:
198
+ task = task or next((t for t in cfg.label_tasks if t in TASKS), "steer_intent")
199
+ today = (now or datetime.now(timezone.utc)).astimezone().date()
200
+ counts = serve(task, window_units(cfg, task, days, src, today), open_browser=open_browser,
201
+ on_ready=on_ready, today=today)
202
+ n = Counter(gold_latest(task).values())
203
+ return {**counts, "task": task, "gold": {c: n[c] for c, _d in TASKS[task].candidates}}
204
+
205
+
206
+ def print_review(report: dict[str, Any]) -> None:
207
+ task = report["task"]
208
+ print(f"label review {task}: {report['answered']} answered, {report['skipped']} skipped; "
209
+ f"gold per label {json.dumps(report['gold'])} (floor {FLOORS[task][0]})")
@@ -0,0 +1,326 @@
1
+ """Apply saved local decision recipes as estimates, separately from calibrated labels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import math
8
+ import re
9
+ import subprocess
10
+ import sys
11
+ import time
12
+ from datetime import datetime, timedelta, timezone
13
+ from pathlib import Path
14
+ from typing import Any, Callable
15
+
16
+ from ..config import AutError, Config, config_path
17
+ from ..event import ActivityEvent, make_event_id
18
+ from ..state import LockBusy, _locked, atomic_write_text, load_json_dict
19
+ from .backends import BackendUnavailable, find_model, make_backend
20
+ from .core import PROMPT_VER, Backend, Choice, Request, calibrate, data_dir, softmax, top_label
21
+ from .tasks import TASKS
22
+
23
+ POLICY = "local-decision-v1"
24
+ TASK_NAMES = {"steer_intent", "work_category", "soft_outcome", "writing_activity"}
25
+
26
+
27
+ def writing_policy_path() -> Path:
28
+ return config_path().with_name("label-local-writing.json")
29
+
30
+
31
+ def reserve_label_time(root: Path, now: datetime) -> str:
32
+ """Reserve a strictly newer local label time across runs and replay jobs."""
33
+ root.mkdir(mode=0o700, parents=True, exist_ok=True)
34
+ path = root / "generation.json"
35
+ with _locked(path):
36
+ atomic_write_text(root / ".gitignore", "*\n")
37
+ previous = -1
38
+ if path.exists():
39
+ saved = load_json_dict(path).get("last_ms")
40
+ if type(saved) is not int or saved < 0:
41
+ raise AutError("label: invalid local generation state; it was not reset")
42
+ previous = saved
43
+ elif any(child.is_dir() for child in root.iterdir()):
44
+ raise AutError("label: missing generation state; it was not reset")
45
+ stamp = max(int(now.timestamp() * 1000), previous + 1)
46
+ try:
47
+ value = (
48
+ datetime(1970, 1, 1, tzinfo=timezone.utc) + timedelta(milliseconds=stamp)
49
+ ).isoformat(timespec="milliseconds")
50
+ except OverflowError as exc:
51
+ raise AutError("label: invalid local generation state; it was not reset") from exc
52
+ atomic_write_text(path, json.dumps({"last_ms": stamp}))
53
+ return value
54
+
55
+
56
+ def _number(value: Any) -> bool:
57
+ try:
58
+ return type(value) in (int, float) and math.isfinite(value)
59
+ except OverflowError:
60
+ return False
61
+
62
+
63
+ def _require(condition: Any) -> None:
64
+ if not condition:
65
+ raise ValueError("invalid local decision policy")
66
+
67
+
68
+ def load_policy(path: Path | None = None) -> tuple[dict[str, Any], str]:
69
+ explicit = path is not None
70
+ path = path or config_path().with_name("label-local.json")
71
+ try:
72
+ obj = json.loads(path.read_text())
73
+ _require(obj["version"] == POLICY and obj["tasks"])
74
+ if "writing_activity" in obj["tasks"]:
75
+ _require(explicit and set(obj["tasks"]) == {"writing_activity"})
76
+ for task, row in obj["tasks"].items():
77
+ _require(task in TASK_NAMES)
78
+ _require(row["prompt_ver"] == f"{task}@{TASKS[task].ver}/p{PROMPT_VER}")
79
+ _require(_number(row["minimum_score"]) and 0.5 <= row["minimum_score"] <= 1)
80
+ _require(re.fullmatch(r"[a-f0-9]{64}", row["source_eval_sha256"]))
81
+ if "binary_decision_threshold" in row:
82
+ boundary = row["binary_decision_threshold"]
83
+ _require(TASKS[task].decision == "boolean" and len(TASKS[task].candidates) == 2)
84
+ _require(isinstance(boundary, dict) and set(boundary) == {"label", "value"})
85
+ _require(isinstance(boundary["label"], str)
86
+ and boundary["label"] in {c for c, _ in TASKS[task].candidates})
87
+ _require(_number(boundary["value"]) and 1e-6 < boundary["value"] < 1 - 1e-6)
88
+ recipe = row["recipe"]
89
+ _require(re.fullmatch(r"[a-zA-Z0-9._-]{1,100}", recipe["id"]))
90
+ _require(recipe["method"] in {"single", "majority", "stack"})
91
+ _require(len(recipe["members"]) in ({1} if recipe["method"] == "single" else {3, 5}))
92
+ _require(len({m["model_id"] for m in recipe["members"]}) == len(recipe["members"]))
93
+ for member in recipe["members"]:
94
+ model = find_model(member["model_id"])
95
+ if model is None:
96
+ raise ValueError("unknown local decision model")
97
+ _require(
98
+ model.backend
99
+ in {"mlx_pplx", "mlx_elephant", "mlx_clef", "mlx_card", "llama_server"}
100
+ )
101
+ _require(member["backend"] == model.backend)
102
+ _require(re.fullmatch(r"[A-Za-z0-9._-]{1,100}", member["model_rev"]))
103
+ _require(_number(member["temperature"]) and member["temperature"] > 0)
104
+ if recipe["method"] == "stack":
105
+ stack = recipe["stack"]
106
+ _require(
107
+ len(stack["a"]) == len(recipe["members"]) and all(map(_number, stack["a"]))
108
+ )
109
+ _require(set(stack["b"]) <= {c for c, _ in TASKS[task].candidates})
110
+ _require(all(map(_number, stack["b"].values())))
111
+ digest = hashlib.sha256(
112
+ json.dumps(obj, sort_keys=True, allow_nan=False).encode()
113
+ ).hexdigest()
114
+ return obj, digest
115
+ except (AssertionError, KeyError, TypeError, ValueError, OSError, AttributeError) as exc:
116
+ raise AutError(
117
+ f"label: invalid or missing {path.name}; local estimates stay disabled"
118
+ ) from exc
119
+
120
+
121
+ def memory_guard() -> int | None:
122
+ """Refuse a local model run below 15% free memory; return the free percentage (None off macOS)."""
123
+ if sys.platform != "darwin":
124
+ return None
125
+ try:
126
+ result = subprocess.run(
127
+ ["/usr/bin/memory_pressure", "-Q"], capture_output=True, text=True, timeout=10
128
+ )
129
+ match = re.search(r"memory free percentage:\s*(\d+)%", result.stdout)
130
+ if result.returncode or not match or int(match[1]) < 15:
131
+ raise BackendUnavailable("label: local model run waits for at least 15% free memory")
132
+ return int(match[1])
133
+ except (OSError, subprocess.SubprocessError) as exc:
134
+ raise BackendUnavailable("label: could not check local model memory headroom") from exc
135
+
136
+
137
+ def combine(
138
+ method: str, dists: list[dict[str, float]], stack: dict[str, Any] | None = None
139
+ ) -> dict[str, float]:
140
+ if method == "single":
141
+ return dists[0]
142
+ labels = list(dists[0])
143
+ if method == "stack" and stack is not None:
144
+ logits = [
145
+ sum(a * math.log(max(d[label], 1e-12)) for a, d in zip(stack["a"], dists))
146
+ + stack["b"].get(label, 0.0)
147
+ for label in labels
148
+ ]
149
+ return dict(zip(labels, softmax(logits)))
150
+ return {label: sum(top_label(d) == label for d in dists) / len(dists) for label in labels}
151
+
152
+
153
+ def score_member(
154
+ member: dict[str, Any],
155
+ requests: list[Request],
156
+ deadline: float,
157
+ factory: Callable[[dict[str, Any]], Backend],
158
+ stopped: Callable[[], bool] | None = None,
159
+ ) -> list[dict[str, float]]:
160
+ def check() -> None:
161
+ if time.monotonic() >= deadline or (stopped is not None and stopped()):
162
+ raise BackendUnavailable("label: local model budget reached; unfinished batch retries")
163
+ memory_guard()
164
+
165
+ check()
166
+ backend = factory(member)
167
+ try:
168
+ check()
169
+ getattr(backend, "prepare", lambda: None)()
170
+ if backend.model_rev != member["model_rev"]:
171
+ raise BackendUnavailable("label: local model revision differs from saved recipe")
172
+ scores = []
173
+ for item in requests:
174
+ check()
175
+ raw = backend.distribution(item)
176
+ if (
177
+ set(raw) != {c["id"] for c in item.candidates}
178
+ or not all(_number(v) and v >= 0 for v in raw.values())
179
+ or sum(raw.values()) <= 0
180
+ ):
181
+ raise BackendUnavailable("label: local model returned an invalid distribution")
182
+ scores.append(calibrate(raw, member["temperature"]))
183
+ return scores
184
+ finally:
185
+ getattr(backend, "close", lambda: None)()
186
+
187
+
188
+ class LocalBackend:
189
+ """Apply the saved recipe, using serial scoring unless execution settings qualify."""
190
+
191
+ backend = "local_decision"
192
+ size_mb: float | None = None
193
+
194
+ def __init__(
195
+ self,
196
+ task: str,
197
+ row: dict[str, Any],
198
+ policy_id: str,
199
+ cfg: Config,
200
+ member_for: Callable[[dict[str, Any]], Backend] | None = None,
201
+ ) -> None:
202
+ self.row, self.recipe, self.policy_id = row, row["recipe"], policy_id
203
+ self.model_id, self.model_rev = "local/" + self.recipe["id"], policy_id
204
+ self.member_for = member_for or (
205
+ lambda m: make_backend(
206
+ task,
207
+ Choice(backend=m["backend"], model_id=m["model_id"], model_rev=m["model_rev"]),
208
+ cfg.label_model,
209
+ cfg.label_llama_server,
210
+ )
211
+ )
212
+ self._requests: list[Request] = []
213
+ self._offset, self._deadline = 0, 0.0
214
+ self._scores: dict[str, dict[str, float]] = {}
215
+ from .local_parallel import load_profile
216
+
217
+ self.runtime = load_profile(task, row) if member_for is None else None
218
+ self.label_model, self.llama_server = cfg.label_model, cfg.label_llama_server
219
+
220
+ @property
221
+ def prefetch_size(self) -> int:
222
+ return self.runtime.group_size if self.runtime is not None else 50
223
+
224
+ def prepare_requests(self, requests: list[Request], deadline: float) -> None:
225
+ self._requests, self._deadline = requests, deadline
226
+ self._offset = 0
227
+ self._scores.clear()
228
+
229
+ def distribution(self, req: Request) -> dict[str, float]:
230
+ if req.unit_id not in self._scores:
231
+ try:
232
+ with _locked(data_dir() / "label" / "model-runtime.json", blocking=False):
233
+ self._score_next(req)
234
+ except LockBusy as exc:
235
+ raise BackendUnavailable("label: local model scoring is already running") from exc
236
+ return self._scores.pop(req.unit_id)
237
+
238
+ def _score_next(self, req: Request) -> None:
239
+ from .local_parallel import MemoryAdmissionError, score_parallel
240
+
241
+ batch = self._requests[self._offset : self._offset + self.prefetch_size]
242
+ if not batch or batch[0].unit_id != req.unit_id:
243
+ raise BackendUnavailable("label: local model request order changed")
244
+ parallel = False
245
+ if self.runtime is not None:
246
+ try:
247
+ scores = score_parallel(
248
+ batch,
249
+ self.row,
250
+ self.label_model,
251
+ self.llama_server,
252
+ self.runtime,
253
+ self._deadline,
254
+ )
255
+ parallel = True
256
+ except MemoryAdmissionError:
257
+ batch = batch[:50]
258
+ print(
259
+ "aut: local parallel scoring waits for memory; using serial groups",
260
+ file=sys.stderr,
261
+ )
262
+ if not parallel:
263
+ members = [
264
+ score_member(m, batch, self._deadline, self.member_for)
265
+ for m in self.recipe["members"]
266
+ ]
267
+ scores = {
268
+ item.unit_id: combine(
269
+ self.recipe["method"], [m[i] for m in members], self.recipe.get("stack")
270
+ )
271
+ for i, item in enumerate(batch)
272
+ }
273
+ if set(scores) != {item.unit_id for item in batch}:
274
+ raise BackendUnavailable("label: saved local recipe returned invalid scores")
275
+ for item in batch:
276
+ dist = scores[item.unit_id]
277
+ if (
278
+ set(dist) != {c["id"] for c in item.candidates}
279
+ or not all(_number(v) and 0 <= v <= 1 for v in dist.values())
280
+ or abs(sum(dist.values()) - 1) > 1e-5
281
+ ):
282
+ raise BackendUnavailable("label: saved local recipe returned invalid scores")
283
+ if boundary := self.row.get("binary_decision_threshold"):
284
+ threshold = boundary["value"]
285
+ scores[item.unit_id] = calibrate(dist, platt={
286
+ "yes": boundary["label"], "a": 1.,
287
+ "b": -math.log(threshold / (1 - threshold)),
288
+ })
289
+ self._scores = scores
290
+ self._offset += len(batch)
291
+
292
+ def close(self) -> None:
293
+ self._scores.clear()
294
+ self._requests.clear()
295
+
296
+
297
+ def decorate(
298
+ event: ActivityEvent, req: Request, backend: LocalBackend, probability: float
299
+ ) -> ActivityEvent:
300
+ attrs = event.attrs
301
+ uncertain = probability <= backend.row["minimum_score"]
302
+ attrs.update(
303
+ task="working.local." + req.task,
304
+ label_policy=POLICY,
305
+ policy_id=backend.policy_id,
306
+ provisional=True,
307
+ human_verified=False,
308
+ calibrated=False,
309
+ native_score=probability,
310
+ source_scope="ongoing_local_eligible",
311
+ population_weight=1,
312
+ label_source="local_decision",
313
+ predicted_label=attrs["label"],
314
+ below_tau=uncertain,
315
+ source_eval_sha256=backend.row["source_eval_sha256"],
316
+ input_hash=hashlib.sha256(json.dumps(req.facts, sort_keys=True).encode()).hexdigest(),
317
+ model_members=[
318
+ {k: m[k] for k in ("model_id", "model_rev")} for m in backend.recipe["members"]
319
+ ],
320
+ )
321
+ if uncertain:
322
+ attrs["label"] = "unknown"
323
+ event.event_id = make_event_id(
324
+ "local-estimate", event.event_id, backend.policy_id, req.unit, attrs["input_hash"]
325
+ )
326
+ return event