tryharness 0.7.22 → 0.7.23

package/lib/speech.mjs

@@ -0,0 +1,480 @@
+import { spawn } from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { MODELS, ensureModels, modelById, modelDir, modelFile, modelsPresent } from "./localmodels.mjs";
+
+// Talk in the buddy, heard on this computer only. Silero VAD finds where someone spoke, so a held key over a fan, a
+// keyboard or silence writes nothing. On a Mac with macOS 26, Apple's own transcriber writes the words (the buddy's
+// HarnessSpeech helper), steered toward the names the relay knows; elsewhere Parakeet does (its feature extractor,
+// encoder and token-and-duration decode in plain JS on ONNX Runtime), and the names the relay knows (its agents, the
+// person's folders and files) replace what it made of them when they sound alike. Moonshine stays for the dictation
+// eval's comparison. The models are "extended": downloaded the first time the person holds Talk, never at setup.
+// Choices measured in the dictation eval (tests/evals/dictation) on synthetic requests and real people reading.
+
+export const RATE = 16_000;
+export const SPEECH_IDS = ["vad", "speech"];
+const FRAME = 512;
+const CONTEXT = 64;
+// Moonshine was trained on clips up to about 30 s, and Parakeet is used as measured: speech is recognised in pieces of
+// at most 25 s, cut between stretches of speech where it can be.
+const MAX_PIECE_SECONDS = 25;
+// The most tokens a second of speech may produce: well above anyone's pace, so a decoder stuck repeating itself stops.
+const TOKENS_PER_SECOND = 8;
+// The most of the relay's names (agents first, then folders and files) Apple's transcriber is steered toward.
+const APPLE_NAMES = 300;
+// The Mac's own word list: on the Apple path, a name of the person's never replaces a word in it.
+let dictionary = null;
+const macWords = () => (dictionary ??= new Set(fs.existsSync("/usr/share/dict/words") ? fs.readFileSync("/usr/share/dict/words", "utf8").split("\n").filter((word) => /^[a-z]{2,}$/.test(word)) : []));
+
+/** A PCM or float WAV (any channel count) as mono samples at its own rate. */
+export function decodeWav(bytes) {
+  const data = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
+  const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
+  const tag = (at) => String.fromCharCode(data[at], data[at + 1], data[at + 2], data[at + 3]);
+  if (data.length < 12 || tag(0) !== "RIFF" || tag(8) !== "WAVE") throw new Error("the recording is not a WAV file");
+  let format = null, body = null;
+  for (let at = 12; at + 8 <= data.length;) {
+    const size = view.getUint32(at + 4, true);
+    if (tag(at) === "fmt ") {
+      const code = view.getUint16(at + 8, true);
+      format = { code: code === 0xfffe ? view.getUint16(at + 32, true) : code, channels: view.getUint16(at + 10, true), rate: view.getUint32(at + 12, true), bits: view.getUint16(at + 22, true) };
+    } else if (tag(at) === "data") body = { at: at + 8, size: Math.min(size, data.length - at - 8) };
+    at += 8 + size + (size & 1);
+  }
+  if (!format || !body || !format.channels || !format.rate) throw new Error("the WAV file has no audio in it");
+  const { code, channels, bits } = format;
+  const width = bits / 8;
+  const read = code === 3 ? (bits === 64 ? (at) => view.getFloat64(at, true) : (at) => view.getFloat32(at, true))
+    : bits === 8 ? (at) => (view.getUint8(at) - 128) / 128
+      : bits === 16 ? (at) => view.getInt16(at, true) / 32_768
+        : bits === 24 ? (at) => ((view.getUint8(at) | (view.getUint8(at + 1) << 8) | (view.getInt8(at + 2) << 16))) / 8_388_608
+          : bits === 32 ? (at) => view.getInt32(at, true) / 2_147_483_648 : null;
+  if (!read || (code !== 1 && code !== 3)) throw new Error(`a ${bits}-bit WAV of format ${code} can't be read`);
+  const frames = Math.floor(body.size / (width * channels));
+  const samples = new Float32Array(frames);
+  for (let i = 0; i < frames; i += 1) {
+    let sum = 0;
+    for (let c = 0; c < channels; c += 1) sum += read(body.at + (i * channels + c) * width);
+    samples[i] = sum / channels;
+  }
+  return { sampleRate: format.rate, samples };
+}
+
+/** 16-bit mono WAV bytes for samples at a rate (the buddy's recordings, the eval's clips). */
+export function encodeWav(samples, sampleRate = RATE) {
+  const out = Buffer.alloc(44 + samples.length * 2);
+  out.write("RIFF", 0); out.writeUInt32LE(36 + samples.length * 2, 4); out.write("WAVE", 8);
+  out.write("fmt ", 12); out.writeUInt32LE(16, 16); out.writeUInt16LE(1, 20); out.writeUInt16LE(1, 22);
+  out.writeUInt32LE(sampleRate, 24); out.writeUInt32LE(sampleRate * 2, 28); out.writeUInt16LE(2, 32); out.writeUInt16LE(16, 34);
+  out.write("data", 36); out.writeUInt32LE(samples.length * 2, 40);
+  for (let i = 0; i < samples.length; i += 1) out.writeInt16LE(Math.max(-32_768, Math.min(32_767, Math.round(samples[i] * 32_767))), 44 + i * 2);
+  return out;
+}
+
+// Windowed-sinc resampling from a table: 8 zero crossings a side, a Blackman window, cut below the lower Nyquist.
+const ZEROS = 8;
+const STEPS = 256;
+const KERNEL = (() => {
+  const table = new Float32Array(ZEROS * STEPS + 1);
+  for (let i = 0; i <= ZEROS * STEPS; i += 1) {
+    const x = i / STEPS;
+    const sinc = x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x);
+    const w = 0.42 + 0.5 * Math.cos((Math.PI * x) / ZEROS) + 0.08 * Math.cos((2 * Math.PI * x) / ZEROS);
+    table[i] = sinc * w;
+  }
+  return table;
+})();
+
+/** Samples at `from` Hz resampled to `to` Hz. */
+export function resample(samples, from, to = RATE) {
+  if (from === to) return samples;
+  const ratio = to / from;
+  const scale = Math.min(1, ratio) * 0.95;
+  const reach = ZEROS / scale;
+  const out = new Float32Array(Math.floor(samples.length * ratio));
+  for (let n = 0; n < out.length; n += 1) {
+    const centre = n / ratio;
+    let sum = 0;
+    for (let k = Math.max(0, Math.ceil(centre - reach)), last = Math.min(samples.length - 1, Math.floor(centre + reach)); k <= last; k += 1) {
+      const at = Math.abs(centre - k) * scale * STEPS;
+      const i = at | 0;
+      if (i < ZEROS * STEPS) sum += samples[k] * (KERNEL[i] + (KERNEL[i + 1] - KERNEL[i]) * (at - i));
+    }
+    out[n] = sum * scale;
+  }
+  return out;
+}
+
+/**
+ * Where speech is, from the VAD's per-frame probabilities: [start, end) in samples. A stretch starts at 0.5 and ends
+ * after 500 ms under 0.35 (a pause between words keeps it going); one shorter than 250 ms is a click, not a word.
+ */
+export function speechSpans(probs, { total = probs.length * FRAME, threshold = 0.5, minSpeechMs = 250, minSilenceMs = 500, padMs = 250 } = {}) {
+  const frameMs = (FRAME / RATE) * 1000;
+  const off = threshold - 0.15;
+  const found = [];
+  let start = -1, quiet = -1;
+  const close = (end) => { if ((end - start) * frameMs >= minSpeechMs) found.push([start * FRAME, end * FRAME]); start = -1; quiet = -1; };
+  for (let i = 0; i < probs.length; i += 1) {
+    if (start < 0) { if (probs[i] >= threshold) start = i; continue; }
+    if (probs[i] >= threshold) quiet = -1;
+    else if (probs[i] < off) {
+      if (quiet < 0) quiet = i;
+      if ((i - quiet) * frameMs >= minSilenceMs) close(quiet);
+    }
+  }
+  if (start >= 0) close(quiet >= 0 ? quiet : probs.length);
+  const pad = Math.round((padMs / 1000) * RATE);
+  const spans = [];
+  for (const [s, e] of found) {
+    const span = [Math.max(0, s - pad), Math.min(total, e + pad)];
+    if (spans.length && span[0] <= spans.at(-1)[1]) spans.at(-1)[1] = span[1];
+    else spans.push(span);
+  }
+  return spans;
+}
+
+/** A Llama-style SentencePiece vocabulary as a decoder: ids to text, special tokens skipped, byte fallback kept. */
+function tokenText(tokenizer) {
+  const pieces = [];
+  for (const [piece, id] of Object.entries(tokenizer.model.vocab)) pieces[id] = piece;
+  const special = new Set(tokenizer.added_tokens.filter((token) => token.special).map((token) => token.id));
+  const decoder = new TextDecoder();
+  const decode = (ids) => {
+    const bytes = [];
+    for (const id of ids) {
+      if (special.has(id) || pieces[id] === undefined) continue;
+      const byte = /^<0x([0-9A-F]{2})>$/.exec(pieces[id]);
+      if (byte) bytes.push(parseInt(byte[1], 16));
+      else bytes.push(...Buffer.from(pieces[id].replaceAll("▁", " "), "utf8"));
+    }
+    return decoder.decode(Uint8Array.from(bytes)).replace(/^ /, "");
+  };
+  // Words the recogniser writes as one piece: the common words a name made of them must not be snapped from.
+  const common = new Set(pieces.filter((piece) => piece?.startsWith("▁") && /^[a-z]{2,}$/i.test(piece.slice(1))).map((piece) => piece.slice(1).toLowerCase()));
+  return { decode, common };
+}
+
+/** NeMo's vocabulary file ("piece id" a line) as a decoder, spaced as NeMo's own detokenizer does. */
+function vocabText(text) {
+  const pieces = text.split("\n").filter(Boolean).map((line) => line.slice(0, line.lastIndexOf(" ")).replaceAll("▁", " "));
+  const decode = (ids) => ids.map((id) => pieces[id]).join("").replace(/^\s|\s\B|(\s)\b/g, (_, space) => (space ? " " : ""));
+  const common = new Set(pieces.filter((piece) => /^ [a-z]{2,}$/i.test(piece)).map((piece) => piece.slice(1).toLowerCase()));
+  return { pieces, decode, common, blank: pieces.indexOf("<blk>") };
+}
+
+/**
+ * Apple's on-device transcriber through the buddy's HarnessSpeech helper (packages/desktop/helper, macOS 26): one process
+ * kept while Talk is in use, a JSON request a line, closed after five idle minutes. Null where there is no helper.
+ */
+export function createAppleSpeech({ helper, locale = Intl.DateTimeFormat().resolvedOptions().locale || "en-US", idleMs = 5 * 60_000, log = () => {} }) {
+  if (!helper || !fs.existsSync(helper)) return null;
+  let child = null, buffer = "", timer = null, chain = Promise.resolve();
+  const waiting = [];
+  const start = () => {
+    const own = (child = spawn(helper, [], { stdio: ["pipe", "pipe", "ignore"] }));
+    own.stdout.setEncoding("utf8");
+    own.stdout.on("data", (chunk) => {
+      buffer += chunk;
+      for (let at; (at = buffer.indexOf("\n")) >= 0; buffer = buffer.slice(at + 1)) waiting.shift()?.(buffer.slice(0, at));
+    });
+    own.on("error", (error) => log(`speech: Apple's transcriber could not start (${error.message})`));
+    own.on("exit", () => { if (child === own) child = null; buffer = ""; for (const answer of waiting.splice(0)) answer(null); });
+  };
+  /** One request, answered in turn: the helper's answer, or { ok: false } when it stopped. */
+  const ask = (request) => (chain = chain.then(() => new Promise((resolve) => {
+    clearTimeout(timer);
+    if (!child) start();
+    waiting.push((line) => { try { resolve(line === null ? { ok: false, error: "Apple's transcriber stopped" } : JSON.parse(line)); } catch { resolve({ ok: false, error: "Apple's transcriber answered something unreadable" }); } });
+    child.stdin.write(`${JSON.stringify({ ...request, locale })}\n`);
+  })).finally(() => { timer = setTimeout(() => child?.stdin.end(), idleMs); timer.unref?.(); }));
+  return { ask, stop: () => { clearTimeout(timer); child?.stdin.end(); } };
+}
+
+export function createSpeech({ inference, root, list = MODELS, vocabulary = () => [], apple = null, fetchImpl, log = () => {} }) {
+  const vadEntry = modelById("vad", list), speechEntry = modelById("speech", list);
+  const fileOf = (entry, pattern) => entry.files.find((file) => pattern.test(file.path))?.path;
+  const engine = speechEntry?.engine ?? "moonshine";
+  let tokens = null, config = null;
+  const load = () => {
+    if (tokens) return;
+    const dir = modelDir(root, speechEntry);
+    if (engine === "parakeet") { tokens = vocabText(fs.readFileSync(path.join(dir, fileOf(speechEntry, /(^|\/)vocab\.txt$/)), "utf8")); return; }
+    tokens = tokenText(JSON.parse(fs.readFileSync(path.join(dir, fileOf(speechEntry, /(^|\/)tokenizer\.json$/)), "utf8")));
+    config = JSON.parse(fs.readFileSync(path.join(dir, fileOf(speechEntry, /(^|\/)config\.json$/)), "utf8"));
+  };
+
+  // ── Apple's transcriber, where this Mac has it ──────────────────────────────
+  // Asked once what it can do; while its model comes down, each status asks again. Without it, Parakeet.
+  let macOS = apple ? null : { available: false };
+  const probe = () => apple.ask({ op: "status" }).then((answer) => { macOS = answer.ok ? answer : macOS ?? { available: false }; });
+  if (apple) void probe();
+  const onMac = () => Boolean(macOS?.available);
+  const ids = () => (onMac() ? ["vad"] : SPEECH_IDS);
+
+  // ── Downloading on first use ────────────────────────────────────────────────
+  let downloading = null, failure = null;
+  const ready = () => modelsPresent(root, ids(), list) && (!onMac() || macOS.installed);
+  const files = () => ids().flatMap((id) => { const entry = modelById(id, list); return entry.files.map((file, index) => ({ file: modelFile(root, entry, index), size: file.size })); });
+  const status = () => {
+    if (apple && macOS === null) return { state: "downloading", percent: 0 };
+    if (ready()) return { state: "ready", model: onMac() ? "Apple's transcriber" : speechEntry.name };
+    if (onMac() && !macOS.installed) {
+      void probe();
+      if (macOS.failed || failure) return { state: "failed", error: macOS.failed ?? failure };
+      return { state: macOS.downloading || downloading ? "downloading" : "absent", percent: macOS.percent ?? 0 };
+    }
+    const all = files();
+    const total = all.reduce((sum, f) => sum + f.size, 0);
+    const have = all.reduce((sum, f) => { for (const candidate of [f.file, `${f.file}.partial`]) { try { return sum + Math.min(f.size, fs.statSync(candidate).size); } catch {} } return sum; }, 0);
+    const percent = Math.min(99, Math.floor((have / total) * 100));
+    if (downloading) return { state: "downloading", percent, bytes: total };
+    return failure ? { state: "failed", error: failure, percent, bytes: total } : { state: "absent", percent, bytes: total };
+  };
+  /** Starts the download if the models are missing; answers where it stands. */
+  const ensure = () => {
+    if (!ready() && !downloading && macOS !== null) {
+      failure = null;
+      log(onMac() ? "speech: getting Apple's transcriber ready" : "speech: downloading the speech models");
+      const mac = onMac() && !macOS.installed ? apple.ask({ op: "install" }).then((answer) => { if (!answer.ok) throw new Error(answer.error); return probe(); }) : null;
+      downloading = Promise.all([ensureModels({ root, ids: ids(), list, ...(fetchImpl ? { fetchImpl } : {}), log }), mac])
+        .then(() => log("speech: ready"), (error) => { failure = error.message; log(`speech: download failed: ${error.message}`); })
+        .finally(() => { downloading = null; });
+    }
+    return status();
+  };
+
+  // ── Hearing ─────────────────────────────────────────────────────────────────
+  const vadFile = fileOf(vadEntry, /\.onnx$/);
+  /** The VAD's speech probability for each 32 ms frame. */
+  const probabilities = (samples, lane) => inference.onnx("vad", async ({ ort, graph }) => {
+    const vad = await graph(vadFile);
+    const rate = new ort.Tensor("int64", BigInt64Array.from([BigInt(RATE)]), []);
+    let state = new ort.Tensor("float32", new Float32Array(2 * 128), [2, 1, 128]);
+    const input = new Float32Array(CONTEXT + FRAME);
+    const probs = new Float32Array(Math.floor(samples.length / FRAME));
+    for (let i = 0; i < probs.length; i += 1) {
+      // Each frame carries the last 64 samples of the one before it, as the model was trained.
+      input.copyWithin(0, FRAME, FRAME + CONTEXT);
+      input.set(samples.subarray(i * FRAME, (i + 1) * FRAME), CONTEXT);
+      const out = await vad.run({ input: new ort.Tensor("float32", input.slice(), [1, CONTEXT + FRAME]), state, sr: rate });
+      probs[i] = out.output.data[0];
+      state = out.stateN;
+    }
+    return probs;
+  }, { lane });
+
+  /** Where in the samples someone spoke: [start, end) pairs, none for noise or silence. */
+  const voiced = async (samples, { lane = "interactive" } = {}) => speechSpans(await probabilities(samples, lane), { total: samples.length });
+
+  /** Moonshine's words for one piece of speech. */
+  const moonshine = (samples, { lane = "interactive" } = {}) => inference.onnx("speech", async ({ ort, graph }) => {
+    load();
+    const [encoder, decoder] = await Promise.all([graph(fileOf(speechEntry, /encoder[^/]*\.onnx$/)), graph(fileOf(speechEntry, /decoder[^/]*\.onnx$/))]);
+    const { last_hidden_state: states } = await encoder.run({ input_values: new ort.Tensor("float32", samples, [1, samples.length]) });
+    const layers = config.decoder_num_hidden_layers, heads = config.decoder_num_key_value_heads, width = config.hidden_size / config.decoder_num_attention_heads;
+    const empty = new ort.Tensor("float32", new Float32Array(0), [1, heads, 0, width]);
+    const past = {};
+    for (let i = 0; i < layers; i += 1) for (const part of ["decoder", "encoder"]) for (const kv of ["key", "value"]) past[`past_key_values.${i}.${part}.${kv}`] = empty;
+    const limit = Math.ceil((samples.length / RATE) * TOKENS_PER_SECOND) + 4;
+    const ids = [];
+    let token = config.decoder_start_token_id;
+    for (let step = 0; step < limit; step += 1) {
+      const out = await decoder.run({ input_ids: new ort.Tensor("int64", BigInt64Array.from([BigInt(token)]), [1, 1]), encoder_hidden_states: states,
+        use_cache_branch: new ort.Tensor("bool", Uint8Array.from([step > 0 ? 1 : 0]), [1]), ...past });
+      const logits = out.logits.data;
+      let best = 0;
+      for (let v = 1; v < logits.length; v += 1) if (logits[v] > logits[best]) best = v;
+      // The encoder's keys and values come from the first step and stay; the decoder's grow by one each step.
+      for (let i = 0; i < layers; i += 1) for (const part of step === 0 ? ["decoder", "encoder"] : ["decoder"]) for (const kv of ["key", "value"]) past[`past_key_values.${i}.${part}.${kv}`] = out[`present.${i}.${part}.${kv}`];
+      if (best === config.eos_token_id) break;
+      ids.push(best);
+      token = best;
+    }
+    return tokens.decode(ids).trim();
+  }, { lane });
+
+  /**
+   * Parakeet's words for one piece of speech: its feature extractor and encoder, then a greedy token-and-duration decode
+   * over the encoder's frames (as NeMo and onnx-asr do): each step names a token (or blank) and how many frames to move.
+   */
+  const parakeet = (samples, { lane = "interactive" } = {}) => inference.onnx("speech", async ({ ort, graph }) => {
+    load();
+    const [features, encoder, joint] = await Promise.all([/nemo\d+\.onnx$/, /encoder[^/]*\.onnx$/, /decoder_joint[^/]*\.onnx$/].map((pattern) => graph(fileOf(speechEntry, pattern))));
+    const count = new ort.Tensor("int64", BigInt64Array.from([BigInt(samples.length)]), [1]);
+    const heard = await features.run({ waveforms: new ort.Tensor("float32", samples, [1, samples.length]), waveforms_lens: count });
+    const { outputs, encoded_lengths } = await encoder.run({ audio_signal: heard.features, length: heard.features_lens });
+    const [, width, frames] = outputs.dims, length = Number(encoded_lengths.data[0]);
+    const frame = (t) => { const v = new Float32Array(width); for (let d = 0; d < width; d += 1) v[d] = outputs.data[d * frames + t]; return v; };
+    const { pieces, blank } = tokens;
+    let states = [1, 2].map(() => new ort.Tensor("float32", new Float32Array(2 * 640), [2, 1, 640]));
+    const ids = [];
+    for (let t = 0, emitted = 0; t < length;) {
+      const out = await joint.run({ encoder_outputs: new ort.Tensor("float32", frame(t), [1, width, 1]), targets: new ort.Tensor("int32", Int32Array.from([ids.at(-1) ?? blank]), [1, 1]),
+        target_length: new ort.Tensor("int32", Int32Array.from([1]), [1]), input_states_1: states[0], input_states_2: states[1] });
+      const scores = out.outputs.data;
+      let token = 0, step = 0;
+      for (let v = 1; v < pieces.length; v += 1) if (scores[v] > scores[token]) token = v;
+      for (let k = 1; k < scores.length - pieces.length; k += 1) if (scores[pieces.length + k] > scores[pieces.length + step]) step = k;
+      if (token !== blank) { states = [out.output_states_1, out.output_states_2]; ids.push(token); emitted += 1; }
+      // At most ten tokens on one frame, as the model's decoding allows.
+      if (step > 0) { t += step; emitted = 0; } else if (token === blank || emitted === 10) { t += 1; emitted = 0; }
+    }
+    return tokens.decode(ids).trim();
+  }, { lane });
+  const recognise = engine === "parakeet" ? parakeet : moonshine;
+
+  /** Speech stretches joined into pieces Moonshine can take, cut between stretches. */
+  const pieces = (samples, spans) => {
+    const out = [];
+    for (const [start, end] of spans) {
+      const last = out.at(-1);
+      if (last && end - last[0] <= MAX_PIECE_SECONDS * RATE) last[1] = end;
+      else for (let s = start; s < end; s += MAX_PIECE_SECONDS * RATE) out.push([s, Math.min(end, s + MAX_PIECE_SECONDS * RATE)]);
+    }
+    return out.map(([start, end]) => samples.subarray(start, end));
+  };
+
+  /** Audio (WAV bytes, or { sampleRate, samples }) to text: { text, heard, ms, model, raw }; nothing heard is { text: "", heard: false }. */
+  const transcribe = async (input, { lane = "interactive" } = {}) => {
+    const started = performance.now();
+    if (!ready()) throw new Error("the speech models are not downloaded yet");
+    const audio = input instanceof Uint8Array || input instanceof ArrayBuffer ? decodeWav(input) : input;
+    const samples = audio.sampleRate === RATE ? Float32Array.from(audio.samples) : resample(audio.samples, audio.sampleRate);
+    const spans = await voiced(samples, { lane });
+    const entries = vocabularyEntries(await vocabulary());
+    if (onMac()) {
+      // Apple's model hears the whole recording, steered toward the names the relay knows; then names are put right as
+      // for Parakeet, with the Mac's own dictionary as the words a name never replaces.
+      if (!spans.length) return { text: "", heard: false, ms: Math.round(performance.now() - started), model: "Apple's transcriber", raw: "" };
+      const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "harness-talk-")), "talk.wav");
+      try {
+        fs.writeFileSync(file, encodeWav(samples), { mode: 0o600 });
+        const answer = await apple.ask({ op: "transcribe", file, names: entries.map((entry) => entry.display).slice(0, APPLE_NAMES) });
+        if (!answer.ok) throw new Error(answer.error);
+        const text = answer.text ? snapNames(answer.text, entries, macWords()).text : "";
+        return { text, heard: Boolean(text), ms: Math.round(performance.now() - started), model: "Apple's transcriber", raw: answer.text };
+      } finally { fs.rmSync(path.dirname(file), { recursive: true, force: true }); }
+    }
+    const said = [];
+    for (const piece of pieces(samples, spans)) { const words = await recognise(piece, { lane }); if (words) said.push(words); }
+    const raw = said.join(" ").trim();
+    load();
+    const text = raw ? snapNames(raw, entries, tokens.common).text : "";
+    return { text, heard: Boolean(text), ms: Math.round(performance.now() - started), model: speechEntry.name, raw };
+  };
+
+  return { ready, status, ensure, transcribe, voiced, recognise, common: () => { load(); return tokens.common; } };
+}
+
+// ── Names ─────────────────────────────────────────────────────────────────────
+// Moonshine cannot be primed with names, so they are fixed afterwards, only where a name is clearly meant: an agent's
+// name (as written or as people say it), a file name where a dot or "dot" was heard, or one word no dictionary has.
+// Words that are the name's own words in plain English ("pick folder", "payout for") are left as said.
+
+const ROSTER = JSON.parse(fs.readFileSync(new URL("./members.json", import.meta.url), "utf8")).members;
+// How people say the names that are not spelled as said.
+const SAID = { bankr: ["banker"], pi: ["pie", "pye"], codex: ["code x", "codecs"], "claude-code": ["cloud code", "clawed code"], "grok-bot": ["grock bot", "grog bot"] };
+const SNAP = 0.8;
+
+const squash = (text) => String(text).toLowerCase().replace(/[^a-z0-9]/g, "");
+function distance(a, b) {
+  let row = Array.from({ length: b.length + 1 }, (_, j) => j);
+  for (let i = 1; i <= a.length; i += 1) {
+    const next = [i];
+    for (let j = 1; j <= b.length; j += 1) next[j] = Math.min(row[j] + 1, next[j - 1] + 1, row[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1));
+    row = next;
+  }
+  return row[b.length];
+}
+/** Metaphone, enough of it to hear "cloud" as "claude" and "demon" as "daemon". */
+export function sound(word) {
+  let w = squash(word).replace(/[0-9]/g, "").replace(/^(kn|gn|pn|ae|wr)/, (m) => m[1]).replace(/^x/, "s").replace(/^wh/, "w").replace(/mb$/, "m");
+  const vowel = (c) => "aeiou".includes(c ?? "");
+  let out = "";
+  for (let i = 0; i < w.length; i += 1) {
+    const c = w[i], next = w[i + 1] ?? "", prev = w[i - 1] ?? "";
+    if (c === prev && c !== "c") continue;
+    if (vowel(c)) { if (i === 0) out += "A"; continue; }
+    if (c === "c") out += next === "h" || (next === "i" && w[i + 2] === "a") ? "X" : "iey".includes(next) ? "S" : "K";
+    else if (c === "d") out += next === "g" && "eiy".includes(w[i + 2] ?? "-") ? "J" : "T";
+    else if (c === "g") { if (next === "h" && !vowel(w[i + 2])) i += 1; else out += "iey".includes(next) ? "J" : "K"; }
+    else if (c === "h") { if (vowel(next) && !"csptg".includes(prev)) out += "H"; }
+    else if (c === "k") { if (prev !== "c") out += "K"; }
+    else if (c === "p") out += next === "h" ? "F" : "P";
+    else if (c === "q") out += "K";
+    else if (c === "s") out += next === "h" || (next === "i" && "oa".includes(w[i + 2] ?? "-")) ? "X" : "S";
+    else if (c === "t") out += next === "h" ? "0" : next === "i" && "oa".includes(w[i + 2] ?? "-") ? "X" : "T";
+    else if (c === "v") out += "F";
+    else if (c === "w" || c === "y") { if (vowel(next)) out += c.toUpperCase(); }
+    else if (c === "x") out += "KS";
+    else if (c === "z") out += "S";
+    else out += c.toUpperCase();
+    if ((c === "c" || c === "s" || c === "p") && next === "h") i += 1;
+  }
+  return out;
+}
+const similar = (a, b) => (2 - distance(a, b) / Math.max(a.length, b.length) - distance(sound(a), sound(b)) / Math.max(1, sound(a).length, sound(b).length)) / 2;
+
+/** What the names are matched by: the agents' names (as written and said), then the person's words, files and folders. */
+export function vocabularyEntries(names = []) {
+  const entries = ROSTER.map((member) => ({ display: member.name, kind: "agent", keys: [...new Set([member.name, ...(SAID[member.id] ?? [])].map(squash))] }));
+  const seen = new Set(entries.map((entry) => squash(entry.display)));
+  for (const name of names) {
+    const display = String(name ?? "").trim();
+    const key = squash(display);
+    if (key.length < 4 || display.length > 60 || /\s/.test(display) || seen.has(key)) continue;
+    seen.add(key);
+    entries.push({ display, kind: /\.[a-z][a-z0-9]{0,4}$/i.test(display) ? "file" : "word", keys: [key] });
+  }
+  return entries;
+}
+
+/**
+ * The text with names put right, and what changed. An agent's name may span words ("cloud code"); a file name needs
+ * the dot that was heard ("store. Mjs", "where dot t s"); any other name replaces one word no dictionary has
+ * ("Fattybuthappy"). Close sounds count (4+ letters, at least 0.8 alike by spelling and Metaphone); `common` is the
+ * recogniser's own one-piece words, which are never taken for a name of the person's.
+ */
+export function snapNames(text, entries, common = new Set()) {
+  const tokens = [...String(text).matchAll(/[A-Za-z0-9]+/g)].map((m) => ({ word: m[0], at: m.index, end: m.index + m[0].length }));
+  const found = [];
+  for (let i = 0; i < tokens.length; i += 1) {
+    for (let n = 1; n <= 6 && i + n <= tokens.length; n += 1) {
+      const window = tokens.slice(i, i + n);
+      const between = String(text).slice(window[0].at, window.at(-1).end);
+      // Words run together only across spaces, commas, dots and hyphens.
+      if (/[^A-Za-z0-9\s,.\-]/.test(between)) break;
+      const dotted = n > 1 && (/\./.test(between) || window.slice(1, -1).some((t) => t.word.toLowerCase() === "dot"));
+      const key = squash(window.filter((t, k) => !(k > 0 && t.word.toLowerCase() === "dot")).map((t) => t.word).join(""));
+      // A file's name before its dot, as heard: one too short to judge by sound ("a dot mjs") must be exact.
+      const stem = dotted ? squash(between.split(/\.|\bdot\b/i)[0]) : key;
+      for (const entry of entries) {
+        if (entry.kind === "file" ? !dotted : entry.kind === "word" ? n > 1 || common.has(key) : n > 3) continue;
+        let score = 0;
+        for (const candidate of entry.keys) {
+          if (candidate === key) { score = 1; break; }
+          if (entry.kind === "file" && (stem.length < 3 || squash(entry.display.replace(/\.[^.]+$/, "")).length < 3)) continue;
+          if (candidate.length < 4 || Math.abs(candidate.length - key.length) > candidate.length * 0.4 || sound(candidate)[0] !== sound(key)[0]) continue;
+          score = Math.max(score, similar(candidate, key));
+        }
+        if (score >= SNAP) found.push({ from: window[0].at, to: window.at(-1).end, display: entry.display, score, span: n });
+      }
+    }
+  }
+  // The closest names first, longer spans before shorter ones; none overlapping.
+  found.sort((a, b) => b.score - a.score || b.span - a.span);
+  const taken = [];
+  for (const match of found) if (!taken.some((t) => match.from < t.to && t.from < match.to)) taken.push(match);
+  taken.sort((a, b) => b.from - a.from);
+  let out = String(text);
+  const changes = [];
+  for (const match of taken) {
+    const before = out.slice(match.from, match.to);
+    if (before === match.display) continue;
+    out = out.slice(0, match.from) + match.display + out.slice(match.to);
+    changes.unshift({ from: before, to: match.display });
+  }
+  return { text: out, changes };
+}

package/lib/store.mjs

@@ -523,8 +523,25 @@ export function openStore(file) {
       return post ? { ...post, mentions } : null;
     },
     roomPosts(goalId, since = 0, limit = 200) {
-      return execute("all", "select id, seq, kind, author, text, mentions, created_at from board_contributions where task_id = ? and seq > ? and kind != 'held' order by seq asc limit ?", goalId, Number(since) || 0, Math.max(1, Math.min(500, Number(limit) || 200)))
-        .map((row) => ({ ...row, seq: Number(row.seq), mentions: row.mentions ? JSON.parse(row.mentions) : [] }));
+      // A file attached in the room names it, so a reader can show the file itself.
+      return execute("all", "select id, seq, kind, author, text, mentions, source, created_at from board_contributions where task_id = ? and seq > ? and kind != 'held' order by seq asc limit ?", goalId, Number(since) || 0, Math.max(1, Math.min(500, Number(limit) || 200)))
+        .map(({ source, ...row }) => ({ ...row, seq: Number(row.seq), mentions: row.mentions ? JSON.parse(row.mentions) : [], ...(row.kind === "file" && String(source).startsWith("file:") ? { file: String(source).slice(5) } : {}) }));
+    },
+    // What each agent's finished runs' receipts showed (receipts.mjs, kept as receipt:<task>): { agent: { tested, untested, failing } }.
+    receiptCounts() {
+      const counts = {};
+      for (const row of execute("all", "select t.target_agent as agent, m.value from board_meta m join board_tasks t on t.id = substr(m.key, 9) where m.key like 'receipt:%' and t.target_agent is not null")) {
+        let verdict = null;
+        try { verdict = JSON.parse(row.value).verdict; } catch {}
+        if (!verdict || verdict === "no changes") continue;
+        const entry = counts[row.agent] ??= { tested: 0, untested: 0, failing: 0 };
+        entry[verdict === "tested" ? "tested" : verdict === "tests failing" ? "failing" : "untested"] += 1;
+      }
+      return counts;
+    },
+    // Posts the guard held (firewall.mjs) since a moment: kept apart from the room, counted for the person's digest.
+    heldCount(goalId, since = "") {
+      return Number(execute("get", "select count(*) as n from board_contributions where task_id = ? and kind = 'held' and created_at > ?", goalId, since).n);
     },
     roomCount(goalId) {
       return Number(execute("get", "select count(*) as n from board_contributions where task_id = ? and kind not in ('held', 'goal')", goalId).n);
@@ -571,10 +588,20 @@ export function openStore(file) {
         order by coalesce(ended_at, started_at, indexed_at) desc limit ?`, Math.max(1, Math.min(100, Number(limit) || 30)))
         .map((row) => ({ ...row, message_count: Number(row.message_count) }));
     },
-    // Each folder an agent's sessions ran in, with its latest, newest first.
-    sessionFolders(limit = 300) {
-      return execute("all", `select agent, cwd, max(coalesce(ended_at, started_at, indexed_at)) as at from sessions where cwd is not null
-        group by agent, cwd order by at desc limit ?`, Math.max(1, Math.min(1000, Number(limit) || 300)));
+    // The relay's own copy of a session (messages with tool calls clipped), for one whose source file is gone.
+    sessionTranscript(agent, sessionId) {
+      return execute("get", "select transcript from sessions where agent = ? and session_id = ?", agent, sessionId)?.transcript ?? "";
+    },
+    // Coding sessions with their source files, oldest first, for the context engine's record of what runs needed.
+    sessionSources({ since = "", limit = 2_000 } = {}) {
+      return execute("all", `select agent, session_id, cwd, started_at, source from sessions
+        where agent in ('claude-code', 'codex') and cwd is not null and source is not null and coalesce(started_at, '') >= ?
+        order by started_at asc limit ?`, since, Math.max(1, Math.min(10_000, Number(limit) || 2_000)));
+    },
+    // Every session's agent, folder, title and when it last worked, newest first.
+    sessionPlaces() {
+      return execute("all", `select agent, cwd, title, coalesce(ended_at, started_at, indexed_at) as at from sessions where cwd is not null
+        order by at desc`);
     },
     // The thread that already carries a session, if any.
     threadForSession(agent, sessionId) {
@@ -733,6 +760,9 @@ export function openStore(file) {
       const row = execute("get", "select * from board_files where id = ?", String(id));
       return row ? { ...row, bytes: row.bytes == null ? null : Number(row.bytes), cloud: Number(row.cloud) === 1, present: Number(row.present) === 1 } : null;
     },
+    setFileCloud(id, cloud) {
+      execute("run", "update board_files set cloud = ? where id = ?", cloud ? 1 : 0, String(id));
+    },
     taskFiles(taskId) {
       return execute("all", "select id from board_files where task_id = ? order by created_at asc", taskId).map((row) => this.file(row.id));
     },

package/lib/swarm.mjs

@@ -52,7 +52,7 @@ export function agentProcessCwds(exclude = new Set()) {
   return cwds;
 }
 
-export function createSwarm({ store, roomAgents, mentionsIn, wakeFor, notifyBoard, goalCommand, stopThreadRuns, runLog, runningPids, provenanceFor, defaults, coordinator, worktrees, roster, authorizeRun, pickable, listModels, folders, runsIn }) {
+export function createSwarm({ store, roomAgents, mentionsIn, wakeFor, notifyBoard, goalCommand, stopThreadRuns, runLog, runningPids, provenanceFor, defaults, coordinator, worktrees, roster, authorizeRun, pickable, listModels, folders, runsIn, threadFiles = () => ({ files: [], rendering: [] }) }) {
   const awaitingStart = (run) => run.status === "open" && Boolean(run.run_workspace) && !run.run_authorized_at;
   const isDir = (folder) => typeof folder === "string" && path.isAbsolute(folder) && fs.statSync(folder, { throwIfNoEntry: false })?.isDirectory();
   const agentsOf = (thread) => thread.goal.team.filter((agent) => agent !== thread.goal.facilitator);
@@ -121,16 +121,18 @@ export function createSwarm({ store, roomAgents, mentionsIn, wakeFor, notifyBoar
   // A thread needs a folder when a coding agent is in it, or when no one is named and the coordinator picks who works.
   const needsFolder = (team) => !team.length || team.some((name) => PROCESS_NAMES[name]);
   // Where each coding agent in a thread would run its turn there: in T3 Code, or here, and why not T3 when the person uses it there.
-  const placeOf = (folder, team, by) => ({ ok: true, folder, name: folder ? path.basename(folder) : null, by,
+  // `lastWorked`: where its agent last worked, when Harness picked another folder from the message, for the person to take.
+  const placeOf = (folder, team, by, lastWorked = null) => ({ ok: true, folder, name: folder ? path.basename(folder) : null, by,
+    lastWorked: lastWorked ? { folder: lastWorked, name: path.basename(lastWorked) } : null,
     runs: folder ? team.filter((name) => PROCESS_NAMES[name]).map((name) => ({ agent: name, ...runsIn(name, folder) })) : [] });
 
-  const read = (thread) => {
+  const read = (thread, hostedCaller = false) => {
     const runs = store.threadRuns(thread.id).map((run) => ({
       id: run.id, agent: run.claimed_by || run.target_agent, kind: run.kind, status: run.status, awaitingStart: awaitingStart(run), startedAt: run.created_at, endedAt: run.reported_at,
       sessionId: run.session_id, folder: run.run_workspace, usage: run.usage, model: run.run_model,
       steps: readRunSteps(runLog(run.id), 300).map((step) => ({ ...step, text: step.text.slice(0, SAY_LIMIT) })),
     }));
-    return { ok: true, thread: summary(thread), posts: store.roomPosts(thread.id, 0, 500), runs };
+    return { ok: true, thread: summary(thread), posts: store.roomPosts(thread.id, 0, 500), runs, ...threadFiles(thread, hostedCaller) };
   };
 
   // `to`: the one agent the person wrote to (the desktop's agent list), so their words go as written, with no @name.
@@ -223,7 +225,7 @@ export function createSwarm({ store, roomAgents, mentionsIn, wakeFor, notifyBoar
         return isDir(folder) ? placeOf(path.resolve(folder), team, "you") : { ok: false, error: `${folder} is not a folder on this computer` };
       }
       if (!needsFolder(team)) return placeOf(null, team, null);
-      return { pending: folders.suggest(text, team).then(({ folder, by }) => placeOf(folder, team, by)) };
+      return { pending: folders.suggest(text, team).then(({ folder, by, lastWorked }) => placeOf(folder, team, by, lastWorked)) };
     }
     // The person's own note on one agent's card ("use it for reviews"), told to every other agent; no text clears it.
     if (command === "board.swarm.note") {
@@ -288,7 +290,7 @@ export function createSwarm({ store, roomAgents, mentionsIn, wakeFor, notifyBoar
     }
     const thread = store.getTask(String(request.id || ""));
     if (!thread || thread.kind !== "goal") return { ok: false, error: "thread not found" };
-    if (command === "board.swarm.read") return read(thread);
+    if (command === "board.swarm.read") return read(thread, request.hostedCaller === true);
     if (command === "board.swarm.diff") return worktrees.diff(thread.id, request.runId ? String(request.runId) : null);
     if (command === "board.swarm.merge" || command === "board.swarm.discard") {
       // Both change the person's repository, so both wait for the person, and never while an agent is changing files.