tryharness 0.7.18 → 0.7.23

package/lib/models.mjs

@@ -0,0 +1,233 @@
+import os from "node:os";
+import { spawn } from "node:child_process";
+import { agentEnvironment } from "./agentnode.mjs";
+import { nameOf } from "./members.mjs";
+import { trackHelper } from "./runner.mjs";
+
+// Which model and effort each coding agent runs with, owned by the relay. Every agent lists its own models (Codex,
+// Pi and OpenCode have a command for it; Claude Code has none, so its list is its own short names plus the Anthropic
+// models the others list here). A run uses, first: a model named for it (`@codex:gpt-6-luna/high` in a post, or
+// post_task's model); then the person's choice for the agent, which may be Auto (Harness's Jev picks from the list
+// against the person's routing words); else nothing, and the agent runs on its own settings. Every pick is checked
+// against the agent's list before a run starts.
+
+const CLAUDE_EFFORTS = ["low", "medium", "high", "xhigh", "max"]; // claude --help
+const PI_THINKING = ["off", "minimal", "low", "medium", "high", "xhigh", "max"]; // pi --help
+const CLAUDE_NAMES = ["fable", "opus", "sonnet", "haiku"];
+/**
+ * The routing words Auto goes by until the person writes their own. The first wording (owner, October 13), "The cheapest
+ * model that can do this work well; more effort only for hard work.", sent demanding work to cheap models (eval
+ * model-routing v1, 20261014-010225-80250750-357a).
+ */
+export const DEFAULT_ROUTING = "Match the model to the work: for easy work, the cheapest model that does it well; for demanding work, the most capable one. Pass over a model that keeps failing here. More effort only for hard work.";
+
+const tokens = (text) => {
+  const match = /^(\d+(?:\.\d+)?)([KM]?)$/i.exec(String(text || ""));
+  return match ? Math.round(Number(match[1]) * ({ k: 1e3, m: 1e6 }[match[2].toLowerCase()] ?? 1)) : null;
+};
+
+// One reader per agent's own list command, its output as the agent prints it today.
+const LISTS = {
+  codex: { command: "codex", args: ["debug", "models"], parse: (out) => JSON.parse(out).models.map((entry) => ({
+    id: entry.slug, about: entry.description || null, efforts: (entry.supported_reasoning_levels || []).map((level) => level.effort),
+    defaultEffort: entry.default_reasoning_level || null, context: entry.context_window || null })) },
+  // provider  model  context  max-out  thinking  images
+  pi: { command: "pi", args: ["--list-models"], parse: (out) => out.split("\n").map((line) => line.trim().split(/\s+/))
+    .filter((cells) => cells.length >= 5 && cells[0] !== "provider")
+    .map(([provider, model, context, , thinking]) => ({ id: `${provider}/${model}`, efforts: thinking === "yes" ? PI_THINKING : [], context: tokens(context) })) },
+  // Each model as "provider/id" on its own line, then its details as JSON.
+  opencode: { command: "opencode", args: ["models", "--verbose"], parse: (out) => {
+    const models = [];
+    let id = null, body = [];
+    const flush = () => {
+      if (!id) return;
+      const entry = JSON.parse(body.join("\n"));
+      models.push({ id, efforts: Object.keys(entry.variants || {}), context: entry.limit?.context ?? null,
+        ...(entry.cost && Number.isFinite(entry.cost.input) ? { price: { inputUsdPerMillion: entry.cost.input, outputUsdPerMillion: entry.cost.output } } : {}) });
+    };
+    for (const line of out.split("\n")) {
+      if (/^[^\s{}"]+\/\S+$/.test(line)) { flush(); id = line.trim(); body = []; } else if (id) body.push(line);
+    }
+    flush();
+    return models;
+  } },
+};
+
+function capture(command, args) {
+  return new Promise((resolve, reject) => {
+    const environment = agentEnvironment(command);
+    if (environment.error) { reject(new Error(environment.error)); return; }
+    let child;
+    try { child = spawn(command, args, { cwd: os.tmpdir(), env: environment.env }); } catch (error) { reject(error); return; }
+    child.stdin.end();
+    trackHelper(child);
+    let out = "", err = "";
+    child.stdout.on("data", (chunk) => { out += chunk; });
+    child.stderr.on("data", (chunk) => { err += chunk; });
+    child.on("error", reject);
+    child.on("close", (code) => code === 0 ? resolve(out) : reject(new Error(`${command} ${args.join(" ")} exited with code ${code}: ${err.trim().slice(0, 200)}`)));
+  });
+}
+
+const catalogKey = (agent) => `models:${agent}`;
+/** An agent's last listing: { at, models, error? }, error when its list command last failed. */
+export const listingOf = (store, agent) => { try { return JSON.parse(store.meta(catalogKey(agent)) || "null"); } catch { return null; } };
+/** An agent's models as it last listed them: [{ id, about?, efforts, defaultEffort?, context?, price? }]. */
+export const catalogOf = (store, agent) => listingOf(store, agent)?.models ?? [];
+
+/**
+ * Asks each installed agent for its models, all at once so one slow agent holds up no other, and keeps the lists; an
+ * agent whose command fails keeps its last list and the error.
+ */
+export async function refreshCatalogs(store, agents, log = () => {}) {
+  await Promise.all(agents.filter((name) => LISTS[name]).map(async (agent) => {
+    try {
+      const models = LISTS[agent].parse(await capture(LISTS[agent].command, LISTS[agent].args)).filter((entry) => entry.id);
+      store.meta(catalogKey(agent), JSON.stringify({ at: new Date().toISOString(), models }));
+    } catch (error) {
+      log(`model list for ${agent}: ${error.message}`);
+      store.meta(catalogKey(agent), JSON.stringify({ ...(listingOf(store, agent) ?? { models: [] }), error: error.message.slice(0, 300) }));
+    }
+  }));
+  if (agents.includes("claude-code")) {
+    const anthropic = [...new Set(["pi", "opencode"].flatMap((other) => catalogOf(store, other)).map((entry) => entry.id).filter((id) => id.startsWith("anthropic/")).map((id) => id.slice("anthropic/".length)))];
+    const models = [...CLAUDE_NAMES.map((name) => ({ id: name, about: `Claude Code's name for its newest ${name[0].toUpperCase()}${name.slice(1)} model.`, efforts: CLAUDE_EFFORTS })),
+      ...anthropic.sort().map((id) => ({ id, efforts: CLAUDE_EFFORTS }))];
+    store.meta(catalogKey("claude-code"), JSON.stringify({ at: new Date().toISOString(), models }));
+  }
+}
+
+/** One model in an agent's list, written as its id or, when only one model has it, the part after the provider. */
+function findModel(store, agent, written) {
+  const models = catalogOf(store, agent);
+  if (!models.length) return { error: `the relay has no model list for ${nameOf(agent)} yet` };
+  const wanted = String(written || "").trim().toLowerCase();
+  const exact = models.find((entry) => entry.id.toLowerCase() === wanted);
+  if (exact) return { entry: exact };
+  const tail = models.filter((entry) => entry.id.toLowerCase().split("/").pop() === wanted);
+  if (tail.length === 1) return { entry: tail[0] };
+  return { error: tail.length ? `"${written}" could be ${tail.map((entry) => entry.id).join(" or ")}; name one` : `"${written}" is not one of ${nameOf(agent)}'s models (npx tryharness agents models ${agent})` };
+}
+
+/** A model and effort for an agent, checked against its list: { ok, pick } with pick { model, effort } or { auto: true }. */
+export function resolvePick(store, agent, model, effort = null) {
+  if (String(model || "").trim().toLowerCase() === "auto") return { ok: true, pick: { auto: true } };
+  const found = findModel(store, agent, model);
+  if (found.error) return { ok: false, error: found.error };
+  const level = effort ? String(effort).trim().toLowerCase() : null;
+  if (level && !found.entry.efforts.includes(level)) {
+    return { ok: false, error: found.entry.efforts.length ? `${found.entry.id} takes effort ${found.entry.efforts.join(", ")}, not "${effort}"` : `${found.entry.id} has no effort setting in ${nameOf(agent)}` };
+  }
+  return { ok: true, pick: { model: found.entry.id, effort: level } };
+}
+
+/** A pick as written after an agent's name in a post, "model", "model/effort" or "auto". */
+export function resolveSpec(store, agent, spec) {
+  const whole = resolvePick(store, agent, spec);
+  if (whole.ok) return whole;
+  const cut = spec.lastIndexOf("/");
+  if (cut > 0) { const split = resolvePick(store, agent, spec.slice(0, cut), spec.slice(cut + 1)); if (split.ok) return split; }
+  return whole;
+}
+
+/** The agents whose model the relay sets. */
+export const PICKABLE = new Set(["claude-code", ...Object.keys(LISTS)]);
+const PICKED = /(?:^|\s)@([a-z][a-z0-9-]*):([^\s]+)/g;
+/** The picks a post names, `@codex:gpt-6-luna/high`, by agent: { picks: Map, error? }. Only agents in `agents` count. */
+export function picksIn(store, text, agents) {
+  const picks = new Map();
+  for (const match of String(text || "").matchAll(PICKED)) {
+    const agent = match[1] === "claude" ? "claude-code" : match[1];
+    if (!agents.includes(agent) || !PICKABLE.has(agent)) continue;
+    const spec = match[2].replace(/[.,;:!?)]+$/, "");
+    const resolved = resolveSpec(store, agent, spec);
+    if (!resolved.ok) return { picks, error: `@${match[1]}:${spec}: ${resolved.error}` };
+    picks.set(agent, resolved.pick);
+  }
+  return { picks };
+}
+
+const choiceKey = (agent) => `model-choice:${agent}`;
+/** The person's choice for an agent: { model, effort }, { auto: true }, or null for the agent's own settings. */
+export const choiceOf = (store, agent) => { try { return JSON.parse(store.meta(choiceKey(agent)) || "null"); } catch { return null; } };
+export function setChoice(store, agent, model, effort = null) {
+  if (!model || String(model).toLowerCase() === "default") { store.meta(choiceKey(agent), ""); return { ok: true, choice: null }; }
+  const resolved = resolvePick(store, agent, model, effort);
+  if (!resolved.ok) return resolved;
+  store.meta(choiceKey(agent), JSON.stringify(resolved.pick));
+  return { ok: true, choice: resolved.pick };
+}
+export const routingOf = (store) => store.meta("routing-note") || null;
+export const setRouting = (store, text) => { const note = String(text || "").replace(/\s+/g, " ").trim().slice(0, 1_000); store.meta("routing-note", note); return note || null; };
+
+/** A pick in words: "gpt-6-luna, effort high", "Auto". */
+export const describePick = (pick) => !pick ? null : pick.auto ? "Auto" : [pick.model, pick.effort && `effort ${pick.effort}`].filter(Boolean).join(", ");
+
+/** The pick named for one run (a post, post_task), kept with the run until it starts. */
+export const setRunPick = (store, runId, pick, by) => store.meta(`model:${runId}`, JSON.stringify({ ...pick, by }));
+const runPick = (store, runId) => { try { return JSON.parse(store.meta(`model:${runId}`) || "null"); } catch { return null; } };
+
+const median = (values) => { const sorted = values.filter(Number.isFinite).sort((a, b) => a - b); return sorted.length ? sorted[Math.floor(sorted.length / 2)] : null; };
+/** How each of an agent's models did on this board: finished, failed, and the median minutes and dollars of a finished run. */
+function recordByModel(store, agent) {
+  const byModel = new Map();
+  for (const row of store.modelWork(agent)) {
+    const entry = byModel.get(row.model) ?? { done: 0, failed: 0, minutes: [], cost: [] };
+    if (row.status === "done") {
+      entry.done++;
+      entry.minutes.push((Date.parse(row.reported_at) - Date.parse(row.started_at)) / 60_000);
+      entry.cost.push(Number(row.usage?.cost_usd));
+    } else if (row.status === "failed") entry.failed++;
+    byModel.set(row.model, entry);
+  }
+  return (model) => {
+    const entry = byModel.get(model);
+    if (!entry) return null;
+    const minutes = median(entry.minutes), cost = median(entry.cost);
+    return { done: entry.done, failed: entry.failed, medianMinutes: minutes === null ? null : Math.round(minutes * 10) / 10, medianCostUsd: cost === null ? null : Math.round(cost * 10_000) / 10_000 };
+  };
+}
+
+/** What Auto sends Harness for one run (src/server/swarm/route.ts): the work, the person's words, every listed model with its record here. */
+export const routeInput = ({ agent, work, transcript, routing, ownModel, models, record }) => ({
+  agent, name: nameOf(agent), work, transcript, routing: routing ?? DEFAULT_ROUTING, ownModel,
+  models: models.slice(0, 250).map((entry) => ({ ...entry, record: record(entry.id) })),
+});
+
+/**
+ * The model and effort a run starts with: { model, effort, by, note? }, or null for the agent's own settings. `by` says
+ * who chose: "you" (the person's post), another agent's id, "your choice" (the person's setting) or "auto".
+ * Auto asks Harness only when the person allows it (not the privacy setting) and Harness can answer.
+ */
+export function createModelRouter({ store, askHarness, canAsk, privacy, ownModel, workOf, log = () => {} }) {
+  return async function modelForRun(task, agent) {
+    const named = runPick(store, task.id);
+    const pick = named ?? (choiceOf(store, agent) ? { ...choiceOf(store, agent), by: "your choice" } : null);
+    if (!pick) return null;
+    const by = named ? (named.by === "user" ? "you" : named.by) : pick.by;
+    if (!pick.auto) {
+      // The list may have changed since the pick: a model the agent no longer lists runs on its own settings, said so.
+      const checked = resolvePick(store, agent, pick.model, pick.effort);
+      return checked.ok ? { ...checked.pick, by } : { model: null, effort: null, by, note: `${checked.error}; it ran on its own settings` };
+    }
+    const fallback = (note) => ({ model: null, effort: null, by: "auto", note });
+    if (privacy()) return fallback("Auto is off on the privacy setting; it ran on its own settings");
+    if (!canAsk()) return fallback("Harness could not pick a model just now; it ran on its own settings");
+    const models = catalogOf(store, agent);
+    if (!models.length) return fallback(`the relay has no model list for ${nameOf(agent)} yet; it ran on its own settings`);
+    try {
+      const reply = await askHarness("route_model", routeInput({ agent, ...workOf(task), routing: routingOf(store), ownModel: ownModel(agent), models, record: recordByModel(store, agent) }), null);
+      if (!reply.ok) throw new Error(reply.error || "Harness did not pick a model");
+      // The bridge hands back Harness's answer with its fields beside ok.
+      const picked = reply;
+      if (!picked.model) return fallback("Harness picked no model; it ran on its own settings");
+      const checked = resolvePick(store, agent, picked.model, picked.effort && models.find((entry) => entry.id === picked.model)?.efforts.includes(picked.effort) ? picked.effort : null);
+      if (!checked.ok) return fallback(`Auto picked ${picked.model}, which ${nameOf(agent)} does not list; it ran on its own settings`);
+      // Jev's confidence in the pick stays with the run, to see later how sure the picks that worked were.
+      return { ...checked.pick, by: "auto", judgments: picked.judgments ?? null, estimatedMicroUsd: picked.estimatedMicroUsd ?? null };
+    } catch (error) {
+      log(`auto model for ${task.id}: ${error.message}`);
+      return fallback(`Harness could not pick a model (${error.message}); it ran on its own settings`);
+    }
+  };
+}

package/lib/network.mjs

@@ -0,0 +1,118 @@
+// Swarms talking to swarms, on the person's computer: the one list every surface shows (the buddy's
+// Swarms tab, the dashboard, the PiP), built here from what Harness holds (who asked whom, and the
+// answer) and what only this computer holds (each thread's words, held replies, what the person has
+// read). A sealed request's subject is opened here, never at Harness. Only the person ("user") calls
+// these; another agent works a thread through board.thread.* on the task it holds.
+
+const SEEN = (id) => `relay_seen:${id}`;
+const AUTO_SEND_DEFAULT = "relay_auto_send_default";
+const DECISIONS = new Set(["accept", "decline", "block"]);
+const handleOf = (identity) => String(identity || "").split("/")[0].toLowerCase();
+const agentOf = (identity) => String(identity || "").split("/")[1] || "";
+
+export function createNetwork({ store, askHarness, relayRequest, openRelayRequest, fingerprint, myKey }) {
+  // Harness answers for the server's half; a refusal comes back as a plain error the person can read.
+  const harness = async (input) => {
+    const answer = await askHarness("relay_network", input);
+    if (!answer.ok) throw new Error(answer.error || "Harness refused the request");
+    const { ok: _ok, ...rest } = answer;
+    return rest;
+  };
+
+  // One row per request or thread, grouped by what it needs: the person (a request to answer, a held reply,
+  // a message from a person), nothing (open), the other side (a request not answered yet), or nobody (ended).
+  const conversations = async () => {
+    const server = await harness({ op: "threads" }).catch(() => null);
+    const local = new Map(store.relayThreads().map((row) => [row.task_id, row]));
+    const items = [];
+    for (const thread of server?.threads ?? []) {
+      const here = local.get(thread.id) || null;
+      local.delete(thread.id);
+      const peer = thread.role === "asker" ? thread.target : thread.asker;
+      let about = thread.about || here?.objective || "";
+      let keyPrint = null;
+      // A sealed request's subject: opened on this computer, with the other computer's key to compare.
+      if (thread.sealed && !about) {
+        const opened = await openRelayRequest({ sealed: thread.sealed, from: `${peer.handle}/${peer.agent}` }).catch(() => null);
+        if (opened) { about = opened.about; keyPrint = opened.fingerprint; }
+      }
+      items.push(item({ id: thread.id, role: thread.role, kind: thread.kind || (thread.target?.agent === "person" ? "person" : "agent"), status: thread.status, about,
+        peer: { handle: peer.handle, agent: peer.agent }, mine: thread.role === "asker" ? thread.asker.agent : thread.target.agent,
+        askerSince: thread.role === "target" ? thread.asker.since : null, sealed: Boolean(thread.sealed), fingerprint: keyPrint, createdAt: thread.createdAt, here }));
+    }
+    // Threads this computer holds that Harness no longer lists (older ones, or Harness out of reach).
+    for (const here of local.values()) {
+      items.push(item({ id: here.task_id, role: here.role, kind: agentOf(here.with_agent) === "person" ? "person" : "agent", status: here.closed_at ? "closed" : "accepted", about: here.objective,
+        peer: { handle: handleOf(here.with_agent), agent: agentOf(here.with_agent) }, mine: here.target_agent === "user" ? "person" : here.target_agent,
+        askerSince: null, sealed: false, fingerprint: null, createdAt: here.opened_at, here }));
+    }
+    items.sort((a, b) => String(b.updatedAt).localeCompare(String(a.updatedAt)));
+    return { ok: true, connected: Boolean(server), items, needsYou: items.filter((entry) => entry.group === "needs_you").length };
+  };
+
+  const item = ({ id, role, kind, status, about, peer, mine, askerSince, sealed, fingerprint: keyPrint, createdAt, here }) => {
+    const seen = Number(store.meta(SEEN(id)) || 0);
+    const thread = here ? {
+      open: !here.closed_at, autoSend: here.auto_send, held: here.held, unread: here.last_in_seq > seen, messages: here.messages, outcome: here.outcome,
+      last: here.last ? { direction: here.last.direction, sender: here.last.sender, text: String(here.last.text || "").slice(0, 160), state: here.last.state, at: here.last.created_at } : null,
+    } : null;
+    const reason = role === "target" && status === "requested" ? "request"
+      : thread?.open && thread.held ? "held"
+      : thread?.open && kind === "person" && thread.unread ? "message"
+      : null;
+    const group = reason ? "needs_you"
+      : thread ? (thread.open ? "open" : "ended")
+      : status === "requested" ? "waiting"
+      : status === "accepted" || status === "accepted_pending" ? "open"
+      : "ended";
+    return { id, role, kind, status, about, peer, mine, askerSince, sealed, fingerprint: keyPrint, group, reason, thread, createdAt, updatedAt: here?.updated_at || createdAt };
+  };
+
+  // A refusal from Harness or the other computer comes back as the answer, in its own words.
+  const answered = (promise) => ({ pending: promise.catch((error) => ({ ok: false, error: error.message })) });
+  const command = (request, agent) => {
+    if (agent !== "user") return { ok: false, error: "only the person talks to other swarms from here" };
+    const id = String(request.id || "");
+    switch (request.command) {
+      case "board.relay.list":
+        return answered(conversations());
+      // The server's half of Swarms (card, search, contacts, blocks): what the dashboard asks Harness directly.
+      case "board.relay.network": {
+        const { command: _command, agent: _agent, ...input } = request;
+        if (input.op === "decide" || input.op === "threads") return { ok: false, error: "use board.relay.decide and board.relay.list" };
+        return answered(harness(input).then((answer) => ({ ok: true, ...answer })));
+      }
+      case "board.relay.decide":
+        if (!DECISIONS.has(String(request.decision))) return { ok: false, error: "decision is accept, decline or block" };
+        return answered(harness({ op: "decide", id, decision: String(request.decision) }).then((answer) => ({ ok: true, ...answer })));
+      // A request from this computer: to one of their listed agents from one of the person's, or to the person themselves.
+      case "board.relay.request": {
+        const handle = String(request.handle || "").toLowerCase().replace(/^@/, "");
+        const toAgent = String(request.toAgent || "person");
+        const fromAgent = toAgent === "person" ? "person" : String(request.fromAgent || "");
+        return answered(relayRequest({ to: `${handle}/${toAgent}`, fromAgent, about: request.about, brief: request.brief, refs: request.refs }).then((made) => ({ ok: true, ...made })));
+      }
+      case "board.relay.seen": {
+        const thread = store.getThread(id);
+        if (!thread) return { ok: false, error: "no thread on this task" };
+        const last = store.threadMessages(id).filter((message) => message.direction === "in").at(-1);
+        store.meta(SEEN(id), String(last?.seq ?? 0));
+        return { ok: true };
+      }
+      // Whether replies the person's agents write wait for the person on a new thread (the default) or send on their own.
+      case "board.relay.defaults":
+        if (typeof request.autoSend === "boolean") store.meta(AUTO_SEND_DEFAULT, request.autoSend ? "1" : "0");
+        return { ok: true, autoSend: store.meta(AUTO_SEND_DEFAULT) === "1" };
+      case "board.relay.peers":
+        return answered(Promise.all(store.relayPeers().map(async (peer) => ({ handle: peer.handle, pinnedAt: peer.pinned_at, fingerprint: await fingerprint(peer.public_key) })))
+          .then(async (peers) => ({ ok: true, peers, mine: myKey() ? await fingerprint(myKey()) : null })));
+      default:
+        return { ok: false, error: `unknown command ${request.command}` };
+    }
+  };
+
+  /** A thread just opened on this computer starts with the person's default for their agents' replies. */
+  const opened = (taskId) => { if (store.meta(AUTO_SEND_DEFAULT) === "1") store.setAutoSend(taskId, true); };
+
+  return { command, opened };
+}

package/lib/nudge.mjs

@@ -9,7 +9,7 @@ import { placeEnv } from "./places.mjs";
 // The session-start nudge: what an interactive agent is told when a session opens, so work
 // waiting on the board is not silent until someone asks. Claude Code runs this as a
 // SessionStart hook (stdout lands in its context); the Pi extension and OpenCode plugin
-// build the same text in-process. Nothing waiting, or no daemon, prints nothing.
+// import this file from the runtime and call the same functions. Nothing waiting, or no daemon, prints nothing.
 
 // The swarm block (cards.mjs), when given, follows the queue: who else is in the person's swarm and what each is for.
 export function nudgeText(agent, tasks, unread = [], goals = [], swarm = "") {
@@ -94,14 +94,21 @@ function agentPids() {
 // Where the session runs, as its own environment says (the agent's, which the hook shares): places.mjs reads it.
 const placeOf = (input) => ({ sessionId: input?.session_id, folder: input?.cwd || process.cwd(), env: placeEnv(process.env) });
 
-export async function notePrompt(agent, input = null) {
-  try { await daemonRequest({ command: "board.prompted", agent, pids: agentPids(), ...(input?.session_id ? placeOf(input) : {}) }, 1_500); } catch {}
+// An extension or plugin running inside the agent passes its own process ids.
+export async function notePrompt(agent, input = null, pids = agentPids()) {
+  try { await daemonRequest({ command: "board.prompted", agent, pids, ...(input?.session_id ? placeOf(input) : {}) }, 1_500); } catch {}
 }
 
 /** A session starting says where it runs. */
-export async function notePlace(agent, input) {
+export async function notePlace(agent, input, pids = agentPids()) {
   if (!input?.session_id) return;
-  try { await daemonRequest({ command: "board.live.place", agent, pids: agentPids(), ...placeOf(input) }, 1_500); } catch {}
+  try { await daemonRequest({ command: "board.live.place", agent, pids, ...placeOf(input) }, 1_500); } catch {}
+}
+
+/** A turn ended with nothing more to do: the session waits for the person, and the buddy can say so. */
+export async function noteTurn(agent, input, pids = agentPids()) {
+  if (!input?.session_id) return;
+  try { await daemonRequest({ command: "board.live.turn", agent, pids, ...placeOf(input) }, 1_500); } catch {}
 }
 
 // What the person sent this session from Harness (daemon board.live.say), taken once.
@@ -139,6 +146,7 @@ if (import.meta.url === `file://${process.argv[1]}`) {
     // The Stop hook: a message waiting for this session keeps the agent going with it.
     const messages = await takeMessages(agent, sessionId);
     if (messages.length) process.stdout.write(JSON.stringify({ decision: "block", reason: messagesText(messages) }) + "\n");
+    else await noteTurn(agent, input);
   } else {
     if (process.argv.includes("--if-changed")) await notePrompt(agent, input);
     else await notePlace(agent, input);

package/lib/ocr.mjs

@@ -0,0 +1,217 @@
+import fs from "node:fs";
+import path from "node:path";
+import pngjs from "pngjs";
+import jpeg from "jpeg-js";
+
+// On-device OCR for images that leave the computer (redact.mjs), with the browser's PP-OCR pair (localmodels.mjs "ocr")
+// ported to plain Node. The detector marks text as a probability map; each connected region of it becomes a box (DB
+// post-processing on axis-aligned regions, the shape screen text has), widened back to the text's full extent. Each
+// box is then read alone at the image's full resolution (per box, as the browser does: batching crops of different
+// widths changed reads), and every character keeps the stretch of the box it was read from, so part of a line can be
+// covered.
+
+// Larger images are refused rather than decoded (a 5 MB PNG can hold far more pixels than any screen).
+export const MAX_PIXELS = 40e6;
+// The detector's own settings (its inference.yml: thresh 0.2, box_thresh 0.45, unclip_ratio 1.4) at a long side of
+// 960 to 2,048 pixels: a retina screenshot is shrunk to 2,048 across, a small image is enlarged.
+const DET = { min: 960, max: 2_048, thresh: 0.2, boxThresh: 0.45, unclip: 1.4, mean: [0.485, 0.456, 0.406], std: [0.229, 0.224, 0.225] };
+const REC = { height: 48, mean: [0.5, 0.5, 0.5], std: [0.5, 0.5, 0.5] };
+// Boxes read per scheduler job, so a long image does not hold the ONNX lane from the firewall's checks.
+const REC_PER_JOB = 24;
+
+export const imageKind = (bytes) => bytes?.length > 8 && bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47 ? "png"
+  : bytes?.length > 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff ? "jpeg" : null;
+
+/** { width, height, data: RGBA } from PNG or JPEG bytes; null for any other format. Throws on a broken or oversized image. */
+export function decodeImage(bytes) {
+  const buffer = Buffer.from(bytes.buffer ?? bytes, bytes.byteOffset ?? 0, bytes.byteLength ?? bytes.length);
+  const kind = imageKind(buffer);
+  if (kind === "png") {
+    if (buffer.readUInt32BE(16) * buffer.readUInt32BE(20) > MAX_PIXELS) throw new Error("the image is too large to check");
+    const { width, height, data } = pngjs.PNG.sync.read(buffer);
+    return { width, height, data };
+  }
+  if (kind === "jpeg") return jpeg.decode(buffer, { useTArray: true, formatAsRGBA: true, maxResolutionInMP: MAX_PIXELS / 1e6, maxMemoryUsageInMB: 1_024 });
+  return null;
+}
+
+/** The recogniser's characters, from the character_dict list in its inference.yml. */
+export function characterList(yml) {
+  const lines = yml.split(/\r?\n/);
+  const out = [];
+  for (const line of lines.slice(lines.findIndex((each) => /^\s*character_dict:\s*$/.test(each)) + 1)) {
+    const item = line.match(/^\s*- (.*)$/);
+    if (!item) break;
+    const value = item[1];
+    out.push(value.startsWith("'") ? value.slice(1, -1).replace(/''/g, "'") : value.startsWith('"') ? JSON.parse(value) : value);
+  }
+  if (!out.length) throw new Error("the recogniser's character list is missing");
+  return out;
+}
+
+// Per axis, the source pixels and weights behind each output pixel: their average when shrinking, bilinear when growing.
+function taps(start, length, count, limit) {
+  const scale = length / count;
+  const at = (p) => Math.min(limit - 1, Math.max(0, p));
+  const out = new Array(count);
+  for (let i = 0; i < count; i += 1) {
+    const list = [];
+    if (scale > 1) {
+      const a = start + i * scale, b = a + scale;
+      for (let p = Math.floor(a); p < b; p += 1) { const w = Math.min(b, p + 1) - Math.max(a, p); if (w > 0) list.push(at(p), w / scale); }
+    } else {
+      const c = start + (i + 0.5) * scale - 0.5, p = Math.floor(c), f = c - p;
+      list.push(at(p), 1 - f, at(p + 1), f);
+    }
+    out[i] = list;
+  }
+  return out;
+}
+
+/** A region of the image resampled to width x height as a normalized BGR tensor (PaddleOCR's models read OpenCV's BGR); transparency is laid on white. */
+function tensorOf(image, [x0, y0, x1, y1], width, height, { mean, std }) {
+  const xs = taps(x0, x1 - x0, width, image.width), ys = taps(y0, y1 - y0, height, image.height);
+  const plane = width * height, out = new Float32Array(3 * plane), data = image.data, stride = image.width * 4;
+  for (let j = 0; j < height; j += 1) {
+    const ty = ys[j];
+    for (let i = 0; i < width; i += 1) {
+      const tx = xs[i];
+      let r = 0, g = 0, b = 0;
+      for (let v = 0; v < ty.length; v += 2) {
+        const row = ty[v] * stride, wy = ty[v + 1];
+        for (let u = 0; u < tx.length; u += 2) {
+          const p = row + tx[u] * 4, w = wy * tx[u + 1], a = data[p + 3] / 255, white = 255 * (1 - a);
+          r += (data[p] * a + white) * w; g += (data[p + 1] * a + white) * w; b += (data[p + 2] * a + white) * w;
+        }
+      }
+      const k = j * width + i;
+      out[k] = (b / 255 - mean[0]) / std[0];
+      out[plane + k] = (g / 255 - mean[1]) / std[1];
+      out[2 * plane + k] = (r / 255 - mean[2]) / std[2];
+    }
+  }
+  return out;
+}
+
+// DB post-processing: the regions above the threshold whose mean probability passes the box threshold, each grown by
+// area x unclip / perimeter on every side (the detector is trained on text shrunk by that much).
+function regions(map, width, height) {
+  const seen = new Uint8Array(width * height), stack = new Int32Array(width * height), boxes = [];
+  for (let start = 0; start < map.length; start += 1) {
+    if (seen[start] || map[start] <= DET.thresh) continue;
+    seen[start] = 1;
+    let top = 0, x0 = width, y0 = height, x1 = -1, y1 = -1, sum = 0, count = 0;
+    stack[top++] = start;
+    const visit = (q) => { if (!seen[q] && map[q] > DET.thresh) { seen[q] = 1; stack[top++] = q; } };
+    while (top) {
+      const p = stack[--top], x = p % width, y = (p - x) / width;
+      sum += map[p]; count += 1;
+      if (x < x0) x0 = x; if (x > x1) x1 = x; if (y < y0) y0 = y; if (y > y1) y1 = y;
+      if (x > 0) visit(p - 1);
+      if (x < width - 1) visit(p + 1);
+      if (y > 0) visit(p - width);
+      if (y < height - 1) visit(p + width);
+    }
+    const w = x1 - x0 + 1, h = y1 - y0 + 1;
+    if (Math.min(w, h) < 3 || sum / count < DET.boxThresh) continue;
+    const d = (w * h * DET.unclip) / (2 * (w + h));
+    boxes.push([x0 - d, y0 - d, x1 + 1 + d, y1 + 1 + d]);
+  }
+  return boxes;
+}
+
+// The detector's memory grows with its input (about 1 KB a pixel: 1.6 GB for a 2048x1280 screenshot, measured
+// October 14), and ONNX Runtime keeps memory for every input shape it has run. So the detector reads full-width bands
+// WINDOW rows tall, always the same shape (rows past the image and columns up to a multiple of 512 repeat its edge), and
+// the probability map is stitched from rows at least MARGIN from a band's edge. Lines are never cut: regions are found
+// on the whole map. On the image-redaction eval's screens this found what one pass did, and the OCR models' memory
+// levelled off at 544 MB instead of growing past 2 GB (160-row bands; 256 rows: 766 MB and about 10% faster).
+const BAND = 160, MARGIN = 32, WINDOW = BAND + 2 * MARGIN;
+
+async function detect(image, { ort, graph }) {
+  const long = Math.max(image.width, image.height);
+  const ratio = Math.min(DET.max, Math.max(DET.min, long)) / long;
+  const width = Math.max(32, Math.round((image.width * ratio) / 32) * 32), height = Math.max(32, Math.round((image.height * ratio) / 32) * 32);
+  const columns = Math.ceil(width / 512) * 512;
+  const session = await graph("det/inference.onnx");
+  const sx = image.width / width, sy = image.height / height;
+  const map = new Float32Array(width * height);
+  for (let top = 0; top < height; top += BAND) {
+    const a = Math.max(0, Math.min(top - MARGIN, height - WINDOW)), rows = Math.min(WINDOW, height - a);
+    const pixels = tensorOf(image, [0, a * sy, image.width, (a + rows) * sy], width, rows, DET), input = new Float32Array(3 * WINDOW * columns);
+    // Padding repeats the edge pixels: a flat block beside the image made the detector see text in empty space.
+    for (let c = 0; c < 3; c += 1) for (let y = 0; y < WINDOW; y += 1) {
+      const from = (c * rows + Math.min(y, rows - 1)) * width, to = (c * WINDOW + y) * columns;
+      input.set(pixels.subarray(from, from + width), to);
+      input.fill(pixels[from + width - 1], to + width, to + columns);
+    }
+    const band = (await session.run({ [session.inputNames[0]]: new ort.Tensor("float32", input, [1, 3, WINDOW, columns]) }))[session.outputNames[0]].data;
+    for (let y = top; y < Math.min(height, top + BAND); y += 1) map.set(band.subarray((y - a) * columns, (y - a) * columns + width), y * width);
+  }
+  return regions(map, width, height)
+    .map(([x0, y0, x1, y1]) => [Math.max(0, x0 * sx), Math.max(0, y0 * sy), Math.min(image.width, x1 * sx), Math.min(image.height, y1 * sy)])
+    .filter(([x0, y0, x1, y1]) => x1 - x0 >= 4 && y1 - y0 >= 4);
+}
+
+// ONNX Runtime keeps memory for every input shape it has run (400 crop widths: 108 MB grew to 513 MB, October 14), so
+// crops are padded on the right to a few widths, with 0 after normalization as PaddleOCR's own batches pad. Over this
+// eval's screens that changed 16 of 845 lines, by spacing or a dash, none of a planted value.
+const bucket = (width) => (width <= 512 ? Math.ceil(width / 32) * 32 : Math.ceil(width / 64) * 64);
+
+// CTC greedy decoding that keeps, for each character, the output steps it was read over (each step is a fixed slice of
+// the crop's width).
+async function recognise(image, box, characters, { ort, graph }) {
+  const session = await graph("rec/inference.onnx");
+  const [x0, y0, x1, y1] = box;
+  const width = Math.max(8, Math.round((REC.height * (x1 - x0)) / (y1 - y0)));
+  const padded = bucket(width), crop = tensorOf(image, box, width, REC.height, REC), tensor = new Float32Array(3 * REC.height * padded);
+  for (let row = 0; row < 3 * REC.height; row += 1) tensor.set(crop.subarray(row * width, (row + 1) * width), row * padded);
+  const input = new ort.Tensor("float32", tensor, [1, 3, REC.height, padded]);
+  const out = (await session.run({ [session.inputNames[0]]: input }))[session.outputNames[0]];
+  const [, all, classes] = out.dims, data = out.data;
+  const steps = Math.min(all, Math.ceil((all * width) / padded));
+  const space = classes === characters.length + 2 ? classes - 1 : -1;
+  const pixels = ((x1 - x0) / width) * (padded / all);
+  const read = [];
+  let total = 0, previous = 0;
+  for (let t = 0; t < steps; t += 1) {
+    let best = 0, p = data[t * classes];
+    for (let c = 1; c < classes; c += 1) if (data[t * classes + c] > p) { p = data[t * classes + c]; best = c; }
+    if (best && best === previous) read[read.length - 1].to = x0 + (t + 1) * pixels;
+    else if (best) { read.push({ ch: best === space ? " " : characters[best - 1] ?? "", from: x0 + t * pixels, to: x0 + (t + 1) * pixels }); total += p; }
+    previous = best;
+  }
+  // One entry per UTF-16 unit, so chars lines up with the text's indexes.
+  const text = read.map((each) => each.ch).join("");
+  const chars = read.flatMap(({ ch, from, to }) => Array.from({ length: ch.length }, () => [Math.floor(from), Math.ceil(to)]));
+  const lead = text.length - text.trimStart().length, kept = text.trim().length;
+  return { text: text.trim(), box: [Math.floor(x0), Math.floor(y0), Math.ceil(x1), Math.ceil(y1)], score: read.length ? total / read.length : 0, chars: chars.slice(lead, lead + kept) };
+}
+
+// Top to bottom, left to right within a row (rows: boxes whose tops are within a quarter of their combined height).
+const readingOrder = (lines) => lines.sort((a, b) => (Math.abs(a.box[1] - b.box[1]) < (a.box[3] - a.box[1] + b.box[3] - b.box[1]) / 4 ? a.box[0] - b.box[0] : a.box[1] - b.box[1]));
+
+export function createOcr({ inference }) {
+  let characters = null;
+  /**
+   * The text in an image: { width, height, lines: [{ text, box: [x0, y0, x1, y1], score, chars: [[x0, x1]] }], ms }, boxes
+   * in the image's pixels. Takes PNG or JPEG bytes or a decoded { width, height, data: RGBA }.
+   */
+  const read = async (input, { lane = "handoff" } = {}) => {
+    const started = performance.now();
+    const image = input?.data && input.width ? input : decodeImage(input);
+    if (!image) throw new Error("only PNG and JPEG images can be read");
+    const boxes = await inference.onnx("ocr", (model) => detect(image, model), { lane });
+    const lines = [];
+    for (let i = 0; i < boxes.length; i += REC_PER_JOB) {
+      lines.push(...await inference.onnx("ocr", async (model) => {
+        characters ||= characterList(fs.readFileSync(path.join(model.dir, "rec", "inference.yml"), "utf8"));
+        const read = [];
+        for (const box of boxes.slice(i, i + REC_PER_JOB)) read.push(await recognise(image, box, characters, model));
+        return read;
+      }, { lane }));
+    }
+    return { width: image.width, height: image.height, lines: readingOrder(lines.filter((line) => line.text)), ms: Math.round(performance.now() - started) };
+  };
+  return { read };
+}