tryharness 0.7.45 → 0.7.54

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/README.md +239 -14
  2. package/assets/dsh-plugin.mjs +72 -0
  3. package/assets/opencode-plugin.js +4 -3
  4. package/cli.mjs +497 -196
  5. package/lib/actorRoster.mjs +28 -5
  6. package/lib/adapters.mjs +165 -222
  7. package/lib/agentRuntimeConfig.mjs +36 -0
  8. package/lib/agentRuntimes.mjs +570 -0
  9. package/lib/agentSignIn.mjs +226 -0
  10. package/lib/atlas.mjs +7 -2
  11. package/lib/audit.mjs +643 -0
  12. package/lib/backupSetup.mjs +141 -0
  13. package/lib/boardAccess.mjs +8 -1
  14. package/lib/boardBackup.mjs +283 -0
  15. package/lib/boardCommands.mjs +5 -10
  16. package/lib/boardFileCommands.mjs +2 -1
  17. package/lib/boardMirror.mjs +523 -0
  18. package/lib/boardProtocol.json +263 -6
  19. package/lib/boardTools.mjs +2 -6
  20. package/lib/boardWaits.mjs +102 -0
  21. package/lib/book.mjs +659 -0
  22. package/lib/bridge.mjs +26 -4
  23. package/lib/browserWake.mjs +60 -0
  24. package/lib/cards.mjs +159 -16
  25. package/lib/cloudBoot.mjs +124 -0
  26. package/lib/commands.mjs +25 -5
  27. package/lib/connectionTools.d.ts +13 -0
  28. package/lib/connectionTools.js +45 -0
  29. package/lib/connectorAuth.mjs +309 -0
  30. package/lib/connectorCatalog.json +41 -0
  31. package/lib/connectorCommands.mjs +121 -0
  32. package/lib/connectorGoogle.mjs +513 -0
  33. package/lib/connectors.mjs +805 -0
  34. package/lib/context.mjs +18 -4
  35. package/lib/contractGate.mjs +377 -0
  36. package/lib/coordinator.mjs +152 -39
  37. package/lib/coreModels.mjs +78 -0
  38. package/lib/daemon.mjs +973 -54
  39. package/lib/desk.mjs +239 -0
  40. package/lib/desktop.mjs +113 -8
  41. package/lib/digest.mjs +32 -0
  42. package/lib/directLink.mjs +283 -0
  43. package/lib/environments.mjs +825 -0
  44. package/lib/folders.mjs +4 -4
  45. package/lib/gitHost.mjs +174 -0
  46. package/lib/goalCommands.mjs +58 -14
  47. package/lib/hookDialects.mjs +298 -0
  48. package/lib/inference.mjs +263 -152
  49. package/lib/landed.mjs +887 -0
  50. package/lib/lessons.mjs +6 -0
  51. package/lib/lineup.mjs +324 -0
  52. package/lib/live.mjs +284 -13
  53. package/lib/localmodels.mjs +10 -0
  54. package/lib/machineIdentity.mjs +3 -0
  55. package/lib/machineRoutes.mjs +40 -2
  56. package/lib/machineRuns.mjs +30 -3
  57. package/lib/machineSeal.mjs +69 -10
  58. package/lib/mcp-stdio.mjs +189 -120
  59. package/lib/mcpRelay.mjs +119 -0
  60. package/lib/members.json +26 -0
  61. package/lib/members.mjs +33 -0
  62. package/lib/modelMemory.mjs +100 -0
  63. package/lib/models.mjs +48 -15
  64. package/lib/museSession.mjs +122 -0
  65. package/lib/nativeHost.mjs +203 -0
  66. package/lib/neovim.mjs +126 -0
  67. package/lib/network.mjs +7 -2
  68. package/lib/nudge.mjs +57 -18
  69. package/lib/onboard.mjs +430 -1
  70. package/lib/oneclawLocal.mjs +150 -0
  71. package/lib/ownMachines.mjs +828 -39
  72. package/lib/pairing.mjs +182 -0
  73. package/lib/paths.mjs +8 -0
  74. package/lib/personModel.mjs +89 -0
  75. package/lib/phones.mjs +119 -3
  76. package/lib/placeCommands.mjs +27 -0
  77. package/lib/places.mjs +337 -9
  78. package/lib/plans.mjs +151 -27
  79. package/lib/profile.mjs +152 -0
  80. package/lib/projects.mjs +624 -137
  81. package/lib/readiness.mjs +400 -0
  82. package/lib/record.mjs +115 -0
  83. package/lib/repoLinks.mjs +354 -0
  84. package/lib/roomChannels.mjs +641 -0
  85. package/lib/runBrief.mjs +32 -2
  86. package/lib/runRequests.mjs +157 -0
  87. package/lib/runWorker.mjs +37 -9
  88. package/lib/runner.mjs +74 -11
  89. package/lib/secretPatterns.json +55 -0
  90. package/lib/secrets.mjs +10 -27
  91. package/lib/seen.mjs +102 -0
  92. package/lib/service.mjs +11 -6
  93. package/lib/sessionControl.mjs +386 -96
  94. package/lib/sessionIndex.mjs +5 -3
  95. package/lib/specialties.json +74 -0
  96. package/lib/specialties.mjs +477 -0
  97. package/lib/specialtyKeys.d.ts +3 -0
  98. package/lib/specialtyKeys.js +58 -0
  99. package/lib/specialtyNotice.mjs +147 -0
  100. package/lib/specialtySnapshot.mjs +138 -0
  101. package/lib/store.mjs +829 -29
  102. package/lib/subagents.mjs +8 -4
  103. package/lib/swarm.mjs +483 -37
  104. package/lib/swarmMemory.mjs +947 -0
  105. package/lib/t3.mjs +56 -17
  106. package/lib/taskCommands.mjs +91 -13
  107. package/lib/taskRuns.mjs +68 -11
  108. package/lib/team.json +253 -144
  109. package/lib/team.mjs +538 -147
  110. package/lib/threadLine.mjs +116 -0
  111. package/lib/threadRuns.mjs +78 -12
  112. package/lib/threadView.mjs +66 -16
  113. package/lib/tokenizer.mjs +405 -0
  114. package/lib/ui.mjs +16 -2
  115. package/lib/usageTotals.mjs +148 -0
  116. package/lib/vault.mjs +1144 -0
  117. package/lib/vaultCloud.mjs +117 -0
  118. package/lib/vaultCommands.mjs +201 -0
  119. package/lib/vaultTools.d.ts +12 -0
  120. package/lib/vaultTools.js +42 -0
  121. package/lib/windowAsks.mjs +42 -0
  122. package/lib/workFocus.mjs +59 -0
  123. package/lib/workLinks.mjs +90 -0
  124. package/lib/worktrees.mjs +51 -13
  125. package/package.json +10 -5
@@ -0,0 +1,400 @@
1
+ import { spawn } from "node:child_process";
2
+ import { createHash } from "node:crypto";
3
+ import fs from "node:fs";
4
+ import path from "node:path";
5
+ import { runProtocolTask } from "./agentRuntimes.mjs";
6
+ import { MACHINE_RUNNERS, member, nameOf } from "./members.mjs";
7
+ import { paths } from "./paths.mjs";
8
+ import { ensureDir, readJson, writeJsonAtomic } from "./files.mjs";
9
+
10
+ // Whether each coding agent on this computer can take a prompt right now: Harness sends it one short prompt and reads
11
+ // the answer. A sign-in file only suggests an agent is signed in; an answer proves it. Setup checks every agent it finds,
12
+ // walks the person through each one that fails, and the daemon checks again when an agent appears or its sign-in
13
+ // changes (createAgentWatch below), so an agent installed later joins the swarm by itself.
14
+
15
+ /**
16
+ * How long one check may take. Measured October 21, 2026 on the owner's Mac (.harness-runs/measure/agent-probe):
17
+ * with the commands below, 12 checks (Claude Code, Codex, Pi, OpenCode, 3 rounds) answered in 2.1-5.2 s; with the
18
+ * fuller run commands, 12 answered in 2.2-24.9 s, the slowest Pi's first start. Signed out, checks failed in
19
+ * 0.2-14.4 s (Codex retries a 401 for 14 s). 60 s is 2.4x the slowest answer seen. A check that reaches it is
20
+ * reported as slow and can be tried again, never as signed out.
21
+ */
22
+ export const CHECK_DEADLINE_MS = 60_000;
23
+
24
+ const PROMPT = "Reply with exactly one word: ready";
25
+
26
+ // Each agent's shortest headless run that still goes through its own sign-in: no tools, no Harness MCP server, and no
27
+ // session saved to the person's history where the agent allows it (OpenCode has no such switch).
28
+ const CHECKS = {
29
+ "claude-code": () => ({
30
+ command: "claude",
31
+ args: [
32
+ "-p",
33
+ PROMPT,
34
+ "--output-format",
35
+ "json",
36
+ "--no-session-persistence",
37
+ "--strict-mcp-config",
38
+ "--tools",
39
+ "",
40
+ ],
41
+ }),
42
+ codex: () => ({
43
+ command: "codex",
44
+ args: [
45
+ "exec",
46
+ "--ephemeral",
47
+ "--skip-git-repo-check",
48
+ "-c",
49
+ 'sandbox_mode="read-only"',
50
+ "-c",
51
+ 'approval_policy="never"',
52
+ "-",
53
+ ],
54
+ stdin: PROMPT,
55
+ }),
56
+ pi: () => ({
57
+ command: "pi",
58
+ args: ["--no-session", "--no-tools", "-p", PROMPT],
59
+ }),
60
+ opencode: () => ({
61
+ command: "opencode",
62
+ args: ["run", "--title", "Harness check", "--", PROMPT],
63
+ }),
64
+ };
65
+
66
+ // An agent Harness runs through its own protocol (ACP, or Muse Code's MSP: `runtime` in members.json, and the person's
67
+ // own ACP agents) takes the same prompt the way it takes work (agentRuntimes.mjs), without Harness's tools. Its session
68
+ // can stay in the agent's own history: neither protocol has a switch to keep one out.
69
+ const byProtocol = (agent) => !CHECKS[agent] && Boolean(member(agent)?.runtime);
70
+
71
+ /** The agents Harness can check by sending them a prompt. */
72
+ export const CHECKABLE = [
73
+ ...Object.keys(CHECKS),
74
+ ...MACHINE_RUNNERS.filter(byProtocol),
75
+ ];
76
+
77
+ /** What the person runs to sign an agent in, as setup and the browser say it. */
78
+ export const SIGN_IN = {
79
+ "claude-code": "claude, then type /login",
80
+ codex: "codex login",
81
+ pi: "pi, then type /login",
82
+ opencode: "opencode auth login",
83
+ };
84
+
85
+ // The words each agent prints when it has no sign-in to use: a missing login, a missing key, or a 401 from its provider.
86
+ // Over ACP and MSP: "Authentication required" (auth_required), Muse Code's authRequired, and Cursor's stale login, which
87
+ // it answers as ordinary text ("Please sign in to continue"; Cursor staff, June 2026).
88
+ const SIGNED_OUT =
89
+ /not logged in|please run \/login|use \/login|no api key|api key not found|missing bearer|401|unauthori[sz]ed|invalid api key|authentication|auth[ _-]?required|please sign in|sign in to continue/i;
90
+
91
+ const lastLine = (text) =>
92
+ String(text)
93
+ .split("\n")
94
+ .map((line) => line.trim())
95
+ .filter(Boolean)
96
+ .at(-1)
97
+ ?.slice(0, 200) ?? "";
98
+
99
+ /** What one check's output means: ready, signed out, slow, or failed with the agent's own last words. */
100
+ export function readCheck({ code, output, timedOut, missing }) {
101
+ if (missing) return { state: "not_installed", detail: null };
102
+ if (timedOut)
103
+ return {
104
+ state: "slow",
105
+ detail: `did not answer within ${CHECK_DEADLINE_MS / 1000} seconds`,
106
+ };
107
+ if (code === 0 && /\bready\b/i.test(output))
108
+ return { state: "ready", detail: null };
109
+ if (SIGNED_OUT.test(output)) return { state: "signed_out", detail: null };
110
+ return { state: "failed", detail: lastLine(output) || `exit ${code}` };
111
+ }
112
+
113
+ /** Send one agent the check prompt in an empty folder and read its answer. */
114
+ export function checkAgent(
115
+ agent,
116
+ {
117
+ cwd = path.join(paths.home, "checks"),
118
+ run = spawn,
119
+ runTask = runProtocolTask,
120
+ deadlineMs = CHECK_DEADLINE_MS,
121
+ } = {}
122
+ ) {
123
+ if (byProtocol(agent))
124
+ return checkByProtocol(agent, { cwd, runTask, deadlineMs });
125
+ const invocation = CHECKS[agent]?.();
126
+ if (!invocation)
127
+ return Promise.resolve({
128
+ state: "unchecked",
129
+ detail: `Harness cannot check ${nameOf(agent)} by itself`,
130
+ ms: 0,
131
+ });
132
+ ensureDir(cwd);
133
+ const started = Date.now();
134
+ return new Promise((resolve) => {
135
+ let output = "";
136
+ let timedOut = false;
137
+ let settled = false;
138
+ const finish = (result) => {
139
+ if (settled) return;
140
+ settled = true;
141
+ clearTimeout(timer);
142
+ resolve({ ...result, ms: Date.now() - started });
143
+ };
144
+ const child = run(invocation.command, invocation.args, {
145
+ cwd,
146
+ stdio: ["pipe", "pipe", "pipe"],
147
+ });
148
+ const timer = setTimeout(() => {
149
+ timedOut = true;
150
+ child.kill("SIGKILL");
151
+ }, deadlineMs);
152
+ child.stdout?.on("data", (chunk) => (output += chunk));
153
+ child.stderr?.on("data", (chunk) => (output += chunk));
154
+ child.stdin?.on("error", () => {});
155
+ child.stdin?.end(invocation.stdin ?? "");
156
+ child.on("error", (error) =>
157
+ finish(
158
+ readCheck({
159
+ output: error.message,
160
+ missing: error.code === "ENOENT",
161
+ })
162
+ )
163
+ );
164
+ child.on("close", (code) => finish(readCheck({ code, output, timedOut })));
165
+ });
166
+ }
167
+
168
+ async function checkByProtocol(agent, { cwd, runTask, deadlineMs }) {
169
+ ensureDir(cwd);
170
+ const started = Date.now();
171
+ const stop = new AbortController();
172
+ let timedOut = false;
173
+ const timer = setTimeout(() => {
174
+ timedOut = true;
175
+ stop.abort();
176
+ }, deadlineMs);
177
+ let answer = null;
178
+ let failure = null;
179
+ try {
180
+ await runTask(
181
+ {
182
+ agent,
183
+ cwd,
184
+ brief: PROMPT,
185
+ title: "Harness check",
186
+ runId: "readiness",
187
+ withoutHarnessTools: true,
188
+ },
189
+ {
190
+ emit: (kind, text) => {
191
+ if (kind === "done") answer = text;
192
+ if (kind === "failed") failure = text;
193
+ },
194
+ observe: () => {},
195
+ signal: stop.signal,
196
+ }
197
+ );
198
+ } catch (error) {
199
+ failure = error.message;
200
+ } finally {
201
+ clearTimeout(timer);
202
+ }
203
+ // What agentRuntimes.mjs says when the agent's command is not on this computer.
204
+ const missing =
205
+ answer === null && /^Install and authenticate/.test(failure ?? "");
206
+ return {
207
+ ...readCheck({
208
+ code: answer === null ? 1 : 0,
209
+ output: answer ?? failure ?? "",
210
+ timedOut,
211
+ missing,
212
+ }),
213
+ ms: Date.now() - started,
214
+ };
215
+ }
216
+
217
+ // An agent signed in where Harness cannot see joins within a quarter of an hour, at the cost of starting a signed-out
218
+ // agent four times an hour.
219
+ export const RECHECK_MS = 15 * 60_000;
220
+
221
+ /**
222
+ * The daemon's record of each agent's last check, kept in a file so setup, the CLI and the browser read the same answer.
223
+ * Each record keeps a stamp of the agent's sign-in files at the time, so a later sign-in is noticed. Two checks of one
224
+ * agent at once share one run.
225
+ */
226
+ export function createAgentReadiness({
227
+ file = path.join(paths.home, "agent-readiness.json"),
228
+ check = checkAgent,
229
+ stampOf = (agent) => signInStamp(agent),
230
+ watchesSignIn = (agent) =>
231
+ signInFiles(agent, process.env.HOME, process.env).length > 0,
232
+ onChange = () => {},
233
+ now = () => new Date(),
234
+ } = {}) {
235
+ const running = new Map();
236
+ const all = () => readJson(file, {});
237
+ const of = (agent) => all()[agent] ?? null;
238
+ const run = (agent) => {
239
+ if (running.has(agent)) return running.get(agent);
240
+ const signIn = stampOf(agent);
241
+ const pending = check(agent)
242
+ .then((result) => {
243
+ const entry = {
244
+ state: result.state,
245
+ detail: result.detail,
246
+ ms: result.ms,
247
+ checkedAt: now().toISOString(),
248
+ signIn,
249
+ };
250
+ writeJsonAtomic(file, { ...all(), [agent]: entry });
251
+ onChange(agent, entry);
252
+ return entry;
253
+ })
254
+ .finally(() => running.delete(agent));
255
+ running.set(agent, pending);
256
+ return pending;
257
+ };
258
+ /** Whether the agent's sign-in files changed since its last check (or it was never checked). */
259
+ const signInChanged = (agent) => of(agent)?.signIn !== stampOf(agent);
260
+ // An agent whose sign-in Harness cannot see (in the keychain, or the person's own ACP agent) is checked again while
261
+ // it is signed out, at most every RECHECK_MS, so signing it in later is noticed. A signed-out check stops before any
262
+ // model is asked, so it costs no tokens.
263
+ const signedOutSince = (agent) => {
264
+ const last = of(agent);
265
+ return last?.state === "signed_out" && !watchesSignIn(agent)
266
+ ? now() - new Date(last.checkedAt)
267
+ : -1;
268
+ };
269
+ /** Whether the agent should be checked now: its sign-in changed, or it is signed out where Harness cannot see. */
270
+ const due = (agent) =>
271
+ signInChanged(agent) || signedOutSince(agent) >= RECHECK_MS;
272
+ return {
273
+ check: run,
274
+ of,
275
+ all,
276
+ signInChanged,
277
+ due,
278
+ checking: (agent) => running.has(agent),
279
+ };
280
+ }
281
+
282
+ // The files an agent's sign-in lives in: when one of them changes, the agent may have just signed in.
283
+ function signInFiles(agent, home, env) {
284
+ const dataHome = env.XDG_DATA_HOME || path.join(home, ".local", "share");
285
+ switch (agent) {
286
+ case "claude-code":
287
+ return [path.join(home, ".claude", ".credentials.json")];
288
+ case "codex":
289
+ return [
290
+ path.join(env.CODEX_HOME || path.join(home, ".codex"), "auth.json"),
291
+ ];
292
+ case "pi":
293
+ return [path.join(home, ".pi", "agent", "auth.json")];
294
+ case "opencode":
295
+ return [path.join(dataHome, "opencode", "auth.json")];
296
+ // Also rewritten when Grok refreshes its token, as Codex's is (xai-org/grok-build, xai-grok-login/src/storage.rs).
297
+ case "grok-build":
298
+ return [
299
+ env.GROK_AUTH_PATH ||
300
+ path.join(env.GROK_HOME || path.join(home, ".grok"), "auth.json"),
301
+ ];
302
+ case "kilo":
303
+ return [path.join(dataHome, "kilo", "auth.json")];
304
+ case "dsh":
305
+ return [
306
+ path.join(env.DSH_HOME || path.join(home, ".dsh"), ".credentials.yaml"),
307
+ ];
308
+ // Devin's token, which does not expire (its auth docs).
309
+ case "devin":
310
+ return [path.join(dataHome, "devin", "credentials.toml")];
311
+ // Cursor, Antigravity, Copilot and Muse Code keep their sign-in in the macOS keychain, with no file that changes
312
+ // only then.
313
+ default:
314
+ return [];
315
+ }
316
+ }
317
+ // Claude Code rewrites ~/.claude.json in every session, its own check included, so the file's time changed every minute
318
+ // and the daemon checked Claude Code once a minute (October 23: 60 checks an hour, each a session counted where Claude
319
+ // Code lives). Only the account in it says a sign-in changed; on a Mac the token itself is in the keychain.
320
+ function claudeAccount(home) {
321
+ try {
322
+ const config = JSON.parse(
323
+ fs.readFileSync(path.join(home, ".claude.json"), "utf8")
324
+ );
325
+ const account = config.oauthAccount ?? {};
326
+ const signedInAs = [
327
+ account.accountUuid,
328
+ account.organizationUuid,
329
+ config.primaryApiKey ? "key" : "",
330
+ ].join("/");
331
+ return createHash("sha256").update(signedInAs).digest("hex").slice(0, 16);
332
+ } catch {
333
+ return "";
334
+ }
335
+ }
336
+
337
+ export function signInStamp(agent, home = process.env.HOME, env = process.env) {
338
+ const fileTimes = signInFiles(agent, home, env).map((file) => {
339
+ try {
340
+ return fs.statSync(file).mtimeMs;
341
+ } catch {
342
+ return 0;
343
+ }
344
+ });
345
+ const account = agent === "claude-code" ? [claudeAccount(home)] : [];
346
+ return [...account, ...fileTimes].join(":");
347
+ }
348
+
349
+ /**
350
+ * Agents installed after setup join the swarm by themselves. Every scan looks at the agents Harness can check that are
351
+ * installed here; one never checked, or whose sign-in files changed since its last check, is checked, and a ready one
352
+ * the person never took out of their swarm is added (`join`, lineup.mjs). Nothing is checked again without a change, so
353
+ * a signed-out agent costs one check, not one a minute, and a daemon restart costs none.
354
+ */
355
+ export function createAgentWatch({
356
+ readiness,
357
+ installed,
358
+ inSwarm,
359
+ removedByPerson,
360
+ join,
361
+ log = () => {},
362
+ everyMs = 60_000,
363
+ }) {
364
+ const scan = async () => {
365
+ for (const agent of installed().filter((id) => CHECKABLE.includes(id))) {
366
+ if (removedByPerson(agent) || readiness.checking(agent)) continue;
367
+ const result = readiness.due(agent)
368
+ ? await readiness.check(agent)
369
+ : readiness.of(agent);
370
+ if (result?.state !== "ready" || inSwarm(agent)) continue;
371
+ const joined = join(agent);
372
+ log(
373
+ joined.ok
374
+ ? `${nameOf(agent)} is ready and joined the swarm`
375
+ : `${nameOf(agent)} is ready but could not join: ${joined.error}`
376
+ );
377
+ }
378
+ };
379
+ let timer = null;
380
+ let scanning = null;
381
+ const tick = () => {
382
+ if (scanning) return scanning;
383
+ scanning = scan()
384
+ .catch((error) => log(`agent watch: ${error.message}`))
385
+ .finally(() => (scanning = null));
386
+ return scanning;
387
+ };
388
+ return {
389
+ scan: tick,
390
+ // Setup has just checked every agent, so the first scan waits one interval and the daemon starts no extra work.
391
+ start() {
392
+ timer = setInterval(tick, everyMs);
393
+ timer.unref?.();
394
+ },
395
+ stop() {
396
+ if (timer) clearInterval(timer);
397
+ timer = null;
398
+ },
399
+ };
400
+ }
package/lib/record.mjs ADDED
@@ -0,0 +1,115 @@
1
+ // The record (docs/swarm-organization.md, "The record"): an agent's judged runs in one specialty on one model. A run
2
+ // counts toward the specialty its step asked for (board_tasks.specialty, written by team.mjs when the step is routed),
3
+ // on the model it ran on (board_tasks.run_model). It is judged by, in order:
4
+ // 1. the person's verdict on it (board_reactions), which overrides everything below;
5
+ // 2. a check of it by an agent on a different model (board_tasks.checked): passed is accepted, sent back rejected.
6
+ // A check on the same model does not count, because models favor their own output;
7
+ // 3. the auditor's code (board_tasks.audit, cli/lib/audit.mjs): a report whose own claims its receipts do not bear out
8
+ // is rejected. A change not committed on the thread's branch is not made, so that mismatch rejects the run even when
9
+ // a check passed it.
10
+ // Silence counts neither way. Being tagged by the person is not a verdict.
11
+
12
+ // At least this many judged runs before a record earns, or questions, a specialty.
13
+ const JUDGED = 3;
14
+ const ACCEPTED = new Set(["approves", "approves_with_changes"]);
15
+
16
+ /** Each run the person judged, by run id: the verdict of their latest post after it and before the agent's next run. */
17
+ function verdictsByRun(store) {
18
+ const reports = new Map();
19
+ for (const run of store.threadReports()) {
20
+ const key = `${run.goal_id}\u0000${run.agent}`;
21
+ if (!reports.has(key)) reports.set(key, []);
22
+ reports.get(key).push(run);
23
+ }
24
+ const verdicts = new Map();
25
+ for (const said of store.verdictsSaid()) {
26
+ const runs = reports.get(`${said.goal_id}\u0000${said.agent}`) ?? [];
27
+ const judged = runs.findLast((run) => run.reported_at <= said.said_at);
28
+ if (judged) verdicts.set(judged.id, said.verdict);
29
+ }
30
+ return verdicts;
31
+ }
32
+
33
+ /** A check by an agent on another model: none of its checkers ran on the run's own model, and one's model is known. */
34
+ const checkedElsewhere = (run) =>
35
+ Boolean(run.checked?.outcome) &&
36
+ Boolean(run.model) &&
37
+ run.checked.by?.some((checker) => checker.model) &&
38
+ !run.checked.by.some((checker) => checker.model === run.model);
39
+
40
+ // The auditor's code (audit.mjs) found a claim of the agent's own its receipts do not bear out. A mismatch against the
41
+ // dispatcher (work assigned while the agent was offline) is not the agent's, and receipts still landing (the claims
42
+ // `waiting` lists) count neither way.
43
+ const auditFailed = (audit) =>
44
+ Boolean(audit) &&
45
+ !audit.waiting?.length &&
46
+ (audit.mismatches ?? []).some((mismatch) => mismatch.against === "agent");
47
+ const notCommitted = (audit) =>
48
+ (audit?.mismatches ?? []).some(
49
+ (mismatch) => mismatch.kind === "not_committed"
50
+ );
51
+
52
+ /** How one run was judged: "accepted", "rejected" or null for silence. */
53
+ function judgmentOf(run, verdict) {
54
+ if (verdict) return ACCEPTED.has(verdict) ? "accepted" : "rejected";
55
+ if (notCommitted(run.audit)) return "rejected";
56
+ if (checkedElsewhere(run))
57
+ return run.checked.outcome === "passed" ? "accepted" : "rejected";
58
+ if (auditFailed(run.audit)) return "rejected";
59
+ return null;
60
+ }
61
+
62
+ /** Every agent's record: agent -> model -> specialty -> { accepted, rejected, judged }. */
63
+ function records(store) {
64
+ const verdicts = verdictsByRun(store);
65
+ const all = new Map();
66
+ for (const run of store.routedRuns()) {
67
+ if (!run.model) continue;
68
+ const judged = judgmentOf(run, verdicts.get(run.id));
69
+ if (!judged) continue;
70
+ const byModel = all.get(run.agent) ?? new Map();
71
+ all.set(run.agent, byModel);
72
+ const bySpecialty = byModel.get(run.model) ?? new Map();
73
+ byModel.set(run.model, bySpecialty);
74
+ const entry = bySpecialty.get(run.specialty) ?? {
75
+ accepted: 0,
76
+ rejected: 0,
77
+ judged: 0,
78
+ };
79
+ entry[judged]++;
80
+ entry.judged++;
81
+ bySpecialty.set(run.specialty, entry);
82
+ }
83
+ return all;
84
+ }
85
+
86
+ /** An agent's record on a model: specialty id -> { accepted, rejected, judged }. */
87
+ export function recordFor(store, agentId, model) {
88
+ return new Map(records(store).get(agentId)?.get(model) ?? []);
89
+ }
90
+
91
+ /** The specialties an agent has earned on a model: at least three judged runs, more accepted than rejected. */
92
+ export function earned(store, agentId, model) {
93
+ return [...recordFor(store, agentId, model)]
94
+ .filter(
95
+ ([, entry]) => entry.judged >= JUDGED && entry.accepted > entry.rejected
96
+ )
97
+ .map(([specialty]) => specialty);
98
+ }
99
+
100
+ /** Records that question a specialty: at least three judged runs, more rejected than accepted. */
101
+ export function underperforming(store) {
102
+ const found = [];
103
+ for (const [agent, byModel] of records(store))
104
+ for (const [model, bySpecialty] of byModel)
105
+ for (const [specialty, entry] of bySpecialty)
106
+ if (entry.judged >= JUDGED && entry.rejected > entry.accepted)
107
+ found.push({
108
+ agent,
109
+ model,
110
+ specialty,
111
+ accepted: entry.accepted,
112
+ judged: entry.judged,
113
+ });
114
+ return found;
115
+ }