tryharness 0.7.22 → 0.7.23

package/lib/redact.mjs

@@ -0,0 +1,334 @@
+import fs from "node:fs";
+import pngjs from "pngjs";
+import { decodeImage, imageKind } from "./ocr.mjs";
+import { secretReason } from "./secrets.mjs";
+import { BASE58ISH, BECH32, EMAIL, EVM, EVM_SHORT, MONEY, PHONE, base58, phoneLike } from "./personal.mjs";
+
+// What an image loses before it leaves the computer for an agent on Harness's side (daemon.mjs, board.file.get): the
+// text in it is read here (ocr.mjs) and every key or password (the handoff firewall's own detection, firewall.mjs),
+// wallet address, private key, seed phrase, amount of money, email address and phone number is covered by a solid box
+// labelled with what was there. A box, not a fill from the surroundings: the reading agent sees that something was
+// removed, and nothing of it is left to reconstruct. Only that part of a line is covered, from where its characters were
+// read, with a margin so no character of it stays legible. The person and the agents on this computer keep the original.
+
+// Bump when what is found or how it is drawn changes: covered copies are kept under this version.
+export const REDACTION_VERSION = 1;
+
+const LABELS = { credential: "key hidden", "private-key": "key hidden", seed: "seed phrase hidden", wallet: "address hidden", money: "amount hidden", email: "email hidden", phone: "phone hidden" };
+const NAMES = {
+  credential: ["a key or password", "keys or passwords"], "private-key": ["a private key", "private keys"], seed: ["a seed phrase", "seed phrases"],
+  wallet: ["a wallet address", "wallet addresses"], money: ["an amount of money", "amounts of money"], email: ["an email address", "email addresses"], phone: ["a phone number", "phone numbers"],
+};
+// When finds of about the same length overlap, the stronger kind names the box (sensitiveSpans).
+const RANK = ["seed", "private-key", "wallet", "credential", "email", "phone", "money"];
+
+// The firewall takes any 64-hex value for a private key. On a screen most are file digests, and a row says which: a
+// sha256: prefix, the "<hex>  <file>" line shasum prints, or a digest or checksum label, with no key named on the row.
+// On such a row nothing is taken for a key on the strength of that value (the firewall may name another token on it).
+const digestRow = (row) => !/private|secret|\bkey\b|wallet|seed|mnemonic/i.test(row) && [...row.matchAll(/(?<![0-9a-fA-F])(?:0x)?[0-9a-fA-F]{64}(?![0-9a-fA-F])/g)]
+  .some((m) => /sha-?(?:256|384|512)[:=]\s*$/i.test(row.slice(0, m.index)) || /^\s+\S+\.\w{1,6}\b/.test(row.slice(m.index + m[0].length)) || /\bsha-?(?:256|384|512)\b|\bdigests?\b|\bchecksums?\b/i.test(row));
+
+// Seed phrases (BIP-39, bip39-english.txt: the official English list, MIT, sha256 2f5eed53…dbda): a run of at least
+// SEED_RUN whitespace-separated list words, nearly all different, in lower or upper case (a capital allowed on the
+// first), with numbering between them however OCR read it ("5.", "5frost", ". tomato"). Words read with one wrong
+// letter still count. Over 2,326 of this repository's text and code files, the only run of eight or more such words was
+// a real test phrase (October 14), so nine is far from prose and below every phrase length.
+const SEED_RUN = 9;
+let wordlist = null;
+const bip39 = () => (wordlist ||= new Set(fs.readFileSync(new URL("./bip39-english.txt", import.meta.url), "utf8").split("\n").map((word) => word.trim()).filter(Boolean)));
+const nearWord = (word) => {
+  const list = bip39();
+  if (list.has(word)) return true;
+  if (word.length < 4) return false;
+  for (const candidate of list) {
+    if (candidate[0] !== word[0] || Math.abs(candidate.length - word.length) > 1) continue;
+    let i = 0, j = 0, edits = 0;
+    while (i < word.length && j < candidate.length && edits <= 1) {
+      if (word[i] === candidate[j]) { i += 1; j += 1; continue; }
+      edits += 1;
+      if (word.length > candidate.length) i += 1; else if (word.length < candidate.length) j += 1; else { i += 1; j += 1; }
+    }
+    if (edits + (word.length - i) + (candidate.length - j) <= 1) return true;
+  }
+  return false;
+};
+
+/** Lines on one visual row joined left to right (two spaces apart, so no pattern runs across boxes), with each line's offset. */
+function rowsOf(lines) {
+  const rows = [];
+  for (const line of [...lines].sort((a, b) => (a.box[1] + a.box[3]) - (b.box[1] + b.box[3]))) {
+    const [, y0, , y1] = line.box;
+    const row = rows.find((each) => Math.min(each.y1, y1) - Math.max(each.y0, y0) >= 0.5 * Math.min(each.y1 - each.y0, y1 - y0));
+    if (row) { row.lines.push(line); row.y0 = Math.min(row.y0, y0); row.y1 = Math.max(row.y1, y1); } else rows.push({ y0, y1, lines: [line] });
+  }
+  return rows.sort((a, b) => a.y0 - b.y0).map((row) => {
+    let text = "";
+    const parts = row.lines.sort((a, b) => a.box[0] - b.box[0]).map((line) => { const at = text ? text.length + 2 : 0; text = text ? `${text}  ${line.text}` : line.text; return { line, at }; });
+    return { text, parts };
+  });
+}
+
+// A value wrapped inside its box (a key in a narrow card): a line ending in a long unbroken token, last on its row and
+// the widest line starting at its left edge nearby, with the line right below starting at that edge and going on in the
+// same kind of characters (hex after hex, lower case after bech32, a digit somewhere after anything else). Read as one
+// line (a dotted name is a file or host, not a value cut short); only finds that cross the break count on it.
+function wrapsOf(lines, rows) {
+  const out = [];
+  const lastOnRow = new Set(rows.map((row) => row.parts.at(-1).line)), firstOnRow = new Set(rows.map((row) => row.parts[0].line));
+  for (const a of lines) {
+    const tail = a.text.match(/\S{16,}$/)?.[0];
+    if (!tail || /\./.test(tail.slice(-16)) || !lastOnRow.has(a)) continue;
+    const h = a.box[3] - a.box[1], char = (a.box[2] - a.box[0]) / a.text.length;
+    const column = lines.filter((each) => Math.abs(each.box[0] - a.box[0]) <= 1.5 * char && Math.abs(each.box[1] - a.box[1]) <= 10 * h);
+    if (column.some((each) => each.box[2] > a.box[2] + 1.5 * char)) continue;
+    const goesOn = /^(?:0x)?[0-9a-fA-F]+$/.test(tail) ? /^[0-9a-fA-F]+$/ : /^[a-z0-9]+$/.test(tail.slice(-16)) ? /^[a-z0-9]+$/ : /^(?=\D*\d)[A-Za-z0-9+/=_-]+$/;
+    const b = column.find((each) => each !== a && firstOnRow.has(each) && each.box[1] - a.box[3] >= -0.3 * h && each.box[1] - a.box[3] <= 0.8 * h
+      && Math.abs(each.box[3] - each.box[1] - h) <= 0.35 * h && goesOn.test(each.text.split(/\s/)[0]));
+    if (b) out.push({ text: a.text + b.text, parts: [{ line: a, at: 0 }, { line: b, at: a.text.length }], cut: a.text.length });
+  }
+  return out;
+}
+
+// A word that names a password or key (not "monkey", "keyboard", "passed" or "author"); a lowercase name of a word,
+// digits and dashes (a pod, image or package: keeper-api-7d9f8c6b5d-x2k9p, ami-0c55b159cbfafe1f0), which the credential
+// model alone sometimes takes for a secret; a value that mixes at least three kinds of character, as passwords do.
+const NAMES_A_SECRET = /passw|passphrase|\bpass\b|secret|(?:\b|_|api)keys?\b|key_|token|credential|\bauth(?:orization|[_-]?token)?\b|pwd|\bpin\b/i;
+const nameLike = (text) => /^[a-z0-9]+(?:[-._][a-z0-9]+)+$/.test(text) && text.split(/[-._]/).some((part) => /^[a-z]{3,}$/.test(part));
+const secretish = (text) => Boolean(secretReason(text)) || (text.length >= 8 && [/[a-z]/, /[A-Z]/, /\d/, /[^A-Za-z0-9]/].filter((re) => re.test(text)).length >= 3);
+// Tokens with a known key shape on their own (secrets.mjs), wherever the firewall's pick on the line fell; a NAME=
+// before one is not part of it (OCR may read its first letter as a digit).
+const shaped = (text) => [...text.matchAll(/\S+/g)].flatMap((m) => {
+  const token = m[0].replace(/^\w[\w.-]*=/, "").replace(/^["'`(<[{]+|["'`)>\]},;.]+$/g, "");
+  const start = m.index + m[0].lastIndexOf(token);
+  return token.length >= 12 && secretReason(token) ? [{ start, end: start + token.length, reason: secretReason(token) }] : [];
+});
+
+// A value read with the label that names it: alone on the line under a short label ("Secret access key", then the
+// key), or in a later cell of a table row whose earlier cell names it ("DEPLOY_TOKEN", then the token).
+function labelsOf(rows) {
+  const out = [];
+  const pair = (name, value) => out.push({ text: `${name}: ${value.text}`, parts: [{ line: value, at: name.length + 2 }], cut: name.length + 2 });
+  rows.forEach((row, i) => {
+    const next = rows[i + 1];
+    if (next && row.parts.length === 1 && next.parts.length === 1) {
+      const label = row.parts[0].line, value = next.parts[0].line, h = label.box[3] - label.box[1];
+      if (label.text.length <= 40 && NAMES_A_SECRET.test(label.text) && Math.abs(value.box[0] - label.box[0]) <= h && value.box[1] - label.box[3] <= 1.2 * h && /^\S{8,}$/.test(value.text)) pair(label.text, value);
+    }
+    row.parts.forEach(({ line }, j) => {
+      const name = line.text.split(/\s+/).find((word) => word.length <= 40 && NAMES_A_SECRET.test(word));
+      if (name) for (const { line: value } of row.parts.slice(j + 1)) if (/^\S{8,}$/.test(value.text)) pair(name, value);
+    });
+  });
+  return out;
+}
+
+const matches = (re, text, keep = () => true) => [...text.matchAll(re)].filter((m) => keep(m[0], text.slice(0, m.index))).map((m) => ({ start: m.index, end: m.index + m[0].length }));
+
+// OCR reads some small lower-case letters as their capitals ("pasSword"), and the credential model leans on such words
+// (0.53 for that line, 0.99 spelled right), so it reads rows with those capitals lowered inside otherwise lower-case
+// words. The length is the same, so what it finds maps back as it is.
+const uncased = (text) => text.replace(/\b[a-z][a-zA-Z]{2,}\b/g, (word) => (/^[a-zCOSUVWXZ]+$/.test(word) ? word.toLowerCase() : word));
+
+// Keys and passwords: the firewall's patterns and credential model per row. A row can hold more than one, and the
+// firewall names one per line, so a row is asked again with what was found blanked, up to three times; a later find
+// counts only if it has a known key shape or mixes upper and lower case letters and digits (not the host or path left
+// beside a password).
+const keyLike = (text) => Boolean(secretReason(text)) || (text.length >= 16 && /[a-z]/.test(text) && /[A-Z]/.test(text) && /\d/.test(text));
+async function credentials(rows, firewall, lane) {
+  const found = [];
+  let asking = rows.map((row, index) => ({ index, text: row.text }));
+  for (let round = 0; round < 3 && asking.length; round += 1) {
+    const spans = await firewall.secrets(asking.map((each) => uncased(each.text)).join("\n"), { lane });
+    const next = new Map();
+    let offset = 0;
+    const starts = asking.map((each) => { const at = offset; offset += each.text.length + 1; return at; });
+    for (const span of spans) {
+      const which = starts.findLastIndex((at) => at <= span.start);
+      const row = asking[which], start = span.start - starts[which], end = Math.min(span.end - starts[which], row.text.length);
+      if (end <= start || found.some((each) => each.row === row.index && each.start < end && start < each.end)) continue;
+      const text = rows[row.index].text, value = row.text.slice(start, end);
+      if ((/32-byte hex key/.test(span.reason) && digestRow(text)) || (round > 0 && !keyLike(value))) continue;
+      if (span.reason === "a credential" && nameLike(value) && !NAMES_A_SECRET.test(text) && !/:\/\/[^\s:@/]+:$/.test(text.slice(0, start))) continue;
+      next.set(row.index, (next.get(row.index) ?? row.text).slice(0, start) + " ".repeat(end - start) + (next.get(row.index) ?? row.text).slice(end));
+      found.push({ row: row.index, start, end, kind: /private key|32-byte hex key/.test(span.reason) ? "private-key" : /seed phrase/.test(span.reason) ? "seed" : "credential" });
+    }
+    asking = [...next].map(([index, text]) => ({ index, text }));
+  }
+  return found;
+}
+
+/** Everything but seed phrases on rows: [{ row, start, end, kind }]. */
+async function rowSpans(rows, firewall, lane) {
+  const found = [];
+  rows.forEach((row, i) => {
+    const add = (spans, kind) => { for (const span of spans) found.push({ row: i, ...span, kind }); };
+    add([...matches(EVM, row.text), ...matches(EVM_SHORT, row.text), ...matches(BECH32, row.text)], "wallet");
+    // Base58 of 50 characters or more is a private key (WIF, Solana's 64-byte keys), shorter an address.
+    for (const span of matches(BASE58ISH, row.text, (token, before) => base58(token) && !/sha\d+[-:][A-Za-z0-9+/]*$/i.test(before))) add([span], span.end - span.start >= 50 ? "private-key" : "wallet");
+    // Not git@host, and not the user:password@host part of a URL.
+    add(matches(EMAIL, row.text, (text, before) => !/^git@/i.test(text) && !/:\/\/\S*$/.test(before)), "email");
+    add(matches(PHONE, row.text, phoneLike), "phone");
+    add(matches(MONEY, row.text, (text) => !/^[-+−]?\s?(?:US)?[$€£¥₹₩₿Ξ]\s?\d$/.test(text)), "money");
+    for (const span of shaped(row.text)) {
+      const key = /private key|32-byte hex key/.test(span.reason);
+      if (!(key && digestRow(row.text))) add([span], key ? "private-key" : "credential");
+    }
+  });
+  return [...found, ...await credentials(rows, firewall, lane)];
+}
+
+// Seed phrase runs over every line in reading order; numbering between words is skipped, anything else ends a run.
+function seedRuns(rows) {
+  const tokens = [];
+  for (const row of rows) for (const { line } of row.parts) {
+    for (const m of line.text.matchAll(/\S+/g)) {
+      if (/^\d{0,2}[.):]?$/.test(m[0])) continue;
+      const word = m[0].match(/^(\d{0,2}[.):]?)([A-Za-z]{3,8})$/);
+      tokens.push(word && nearWord(word[2].toLowerCase()) ? { line, start: m.index + word[1].length, end: m.index + m[0].length, word: word[2] } : null);
+    }
+  }
+  const runs = [];
+  let run = [];
+  const close = () => {
+    while (run.length > 1 && /^[A-Z][a-z]/.test(run.at(-1).word)) run.pop();
+    const distinct = new Set(run.map((each) => each.word.toLowerCase())).size;
+    if (Math.min(run.length, distinct + 1) >= SEED_RUN && !run.slice(1).some((each) => /^[A-Z][a-z]/.test(each.word))) runs.push(run);
+    run = [];
+  };
+  for (const token of tokens) { if (token) run.push(token); else close(); }
+  close();
+  return runs;
+}
+
+/** The sensitive spans in what OCR read: [{ line, start, end, kind, item }], one item per thing found. */
+export async function sensitiveSpans(lines, { firewall, lane = "handoff" }) {
+  const rows = rowsOf(lines), wraps = wrapsOf(lines, rows);
+  const found = [];
+  let item = 0;
+  // A span of a row becomes a span of each line it crosses.
+  const place = (row, start, end, kind) => {
+    const id = item++;
+    for (const { line, at } of row.parts) {
+      const from = Math.max(start, at) - at, to = Math.min(end, at + line.text.length) - at;
+      if (to > from) found.push({ line, start: from, end: to, kind, item: id });
+    }
+  };
+  for (const span of await rowSpans(rows, firewall, lane)) place(rows[span.row], span.start, span.end, span.kind);
+  if (wraps.length) for (const span of await rowSpans(wraps, firewall, lane)) if (span.start < wraps[span.row].cut && wraps[span.row].cut < span.end) place(wraps[span.row], span.start, span.end, span.kind);
+  const labels = labelsOf(rows);
+  // Only a value like a password or a known key counts there ("production" beside DEPLOY_TOKEN does not).
+  if (labels.length) for (const span of await credentials(labels, firewall, lane)) if (span.start >= labels[span.row].cut && secretish(labels[span.row].text.slice(span.start, span.end))) place(labels[span.row], span.start, span.end, span.kind);
+  for (const run of seedRuns(rows)) { const id = item++; for (const token of run) found.push({ line: token.line, start: token.start, end: token.end, kind: "seed", item: id }); }
+  // Two-factor recovery and backup codes each work as a password, and look like nothing else: after a heading that names
+  // them, every code-shaped token (letters and digits, in dashed groups or 8 to 16 long) is covered, row by row, until a
+  // row holds none (eval image-redaction: the credential model scored them 0.03 to 0.05).
+  for (const [index, row] of rows.entries()) {
+    if (!/\b(?:recovery|backup)\s+codes?\b/i.test(row.text)) continue;
+    for (let next = index + 1, seen = false; next < Math.min(rows.length, index + 8); next += 1) {
+      const codes = [...rows[next].text.matchAll(/\b(?=[a-z0-9-]*\d)(?=[a-z0-9-]*[a-z])(?:[a-z0-9]{3,8}(?:-[a-z0-9]{3,8})+|[a-z0-9]{8,16})\b/gi)];
+      if (!codes.length) { if (seen) break; continue; }
+      seen = true;
+      for (const code of codes) place(rows[next], code.index, code.index + code[0].length, "credential");
+    }
+  }
+  // Overlapping spans on a line merge, named by the longer find (a key holding an address-like stretch is a key), the
+  // stronger kind when about as long.
+  const merged = [];
+  for (const span of found.sort((a, b) => a.start - b.start)) {
+    const same = merged.find((each) => each.line === span.line && each.start < span.end && span.start < each.end);
+    if (!same) { merged.push({ ...span, length: span.end - span.start }); continue; }
+    const length = span.end - span.start;
+    if (length > same.length + 2 || (Math.abs(length - same.length) <= 2 && RANK.indexOf(span.kind) < RANK.indexOf(same.kind))) Object.assign(same, { kind: span.kind, item: span.item, length });
+    same.start = Math.min(same.start, span.start); same.end = Math.max(same.end, span.end);
+  }
+  return merged.map(({ length, ...span }) => span);
+}
+
+/**
+ * The box covering characters start..end of a line: from halfway between neighbouring characters (all of a space beside
+ * it, where OCR places the space unevenly), with half a character's margin, so no part of a covered character shows.
+ */
+function boxFor({ line, start, end }, width, height) {
+  const { box: [x0, y0, x1, y1], chars, text } = line;
+  const n = text.length;
+  const known = chars?.length === n;
+  const edge = (k, side) => {
+    if (k <= 0) return x0;
+    if (k >= n) return x1;
+    if (!known) return x0 + ((x1 - x0) * k) / n;
+    if (side === "start" && /\s/.test(text[k - 1])) return chars[k - 1][0];
+    if (side === "end" && /\s/.test(text[k])) return chars[k][1];
+    return (chars[k - 1][1] + chars[k][0]) / 2;
+  };
+  const pad = Math.max(2, ((x1 - x0) / Math.max(1, n)) * 0.5), lift = (y1 - y0) * 0.08;
+  return [Math.max(0, Math.floor(edge(start, "start") - pad)), Math.max(0, Math.floor(y0 - lift)), Math.min(width, Math.ceil(edge(end, "end") + pad)), Math.min(height, Math.ceil(y1 + lift))];
+}
+
+// A 5x7 pixel font with only the letters the labels use.
+const GLYPHS = {
+  a: ".....|.....|.###.|....#|.####|#...#|.####", d: "....#|....#|.####|#...#|#...#|#...#|.####", e: ".....|.....|.###.|#...#|#####|#....|.###.",
+  h: "#....|#....|#.##.|##..#|#...#|#...#|#...#", i: "..#..|.....|.##..|..#..|..#..|..#..|.###.", k: "#....|#....|#..#.|#.#..|##...|#.#..|#..#.",
+  l: ".##..|..#..|..#..|..#..|..#..|..#..|.###.", m: ".....|.....|##.#.|#.#.#|#.#.#|#.#.#|#.#.#", n: ".....|.....|#.##.|##..#|#...#|#...#|#...#",
+  o: ".....|.....|.###.|#...#|#...#|#...#|.###.", p: ".....|.....|####.|#...#|####.|#....|#....", r: ".....|.....|#.##.|##..#|#....|#....|#....",
+  s: ".....|.....|.####|#....|.###.|....#|####.", t: ".#...|.#...|####.|.#...|.#...|.#..#|..##.", u: ".....|.....|#...#|#...#|#...#|#..##|.##.#",
+  y: ".....|.....|#...#|#...#|.####|....#|.###.", " ": ".....|.....|.....|.....|.....|.....|.....",
+};
+// Dark boxes on light screens and light ones on dark screens, so the box itself is always seen.
+const DARK = [28, 28, 30], LIGHT = [214, 214, 218];
+
+function paint(image, [x0, y0, x1, y1], label) {
+  const { data, width } = image;
+  let sum = 0;
+  for (let y = y0; y < y1; y += 1) for (let x = x0; x < x1; x += 1) { const p = (y * width + x) * 4; sum += 0.3 * data[p] + 0.59 * data[p + 1] + 0.11 * data[p + 2]; }
+  const [FILL, INK] = sum / Math.max(1, (x1 - x0) * (y1 - y0)) < 110 ? [LIGHT, DARK] : [DARK, LIGHT];
+  for (let y = y0; y < y1; y += 1) for (let x = x0; x < x1; x += 1) { const p = (y * width + x) * 4; data[p] = FILL[0]; data[p + 1] = FILL[1]; data[p + 2] = FILL[2]; data[p + 3] = 255; }
+  // The longest label that fits, at the largest whole scale up to about half the box's height; none if nothing fits.
+  for (const text of [label, "hidden"]) {
+    for (let scale = Math.max(1, Math.floor(((y1 - y0) * 0.55) / 7)); scale >= 1; scale -= 1) {
+      const w = (text.length * 6 - 1) * scale, h = 7 * scale;
+      if (w > x1 - x0 - 2 * scale || h > y1 - y0) continue;
+      const left = x0 + Math.floor((x1 - x0 - w) / 2), top = y0 + Math.floor((y1 - y0 - h) / 2);
+      [...text].forEach((ch, i) => GLYPHS[ch].split("|").forEach((row, gy) => [...row].forEach((on, gx) => {
+        if (on !== "#") return;
+        for (let dy = 0; dy < scale; dy += 1) for (let dx = 0; dx < scale; dx += 1) {
+          const p = ((top + gy * scale + dy) * width + left + (i * 6 + gx) * scale + dx) * 4;
+          data[p] = INK[0]; data[p + 1] = INK[1]; data[p + 2] = INK[2];
+        }
+      })));
+      return;
+    }
+  }
+}
+
+/** "a key or password, a wallet address and 2 amounts of money" for what redactImage covered. */
+export function describeMasked(masked) {
+  const items = new Map();
+  for (const entry of masked) items.set(entry.item, entry.kind);
+  const counts = new Map();
+  for (const kind of items.values()) counts.set(kind, (counts.get(kind) ?? 0) + 1);
+  const parts = RANK.filter((kind) => counts.has(kind)).map((kind) => (counts.get(kind) === 1 ? NAMES[kind][0] : `${counts.get(kind)} ${NAMES[kind][1]}`));
+  return { count: items.size, text: parts.length > 1 ? `${parts.slice(0, -1).join(", ")} and ${parts.at(-1)}` : parts[0] ?? "" };
+}
+
+/**
+ * The image with what must not leave the computer covered: { bytes, mime, masked: [{ kind, box, label, item }], ms },
+ * the original bytes when nothing was found, or { refused } when it cannot be checked (a format other than PNG or JPEG,
+ * an animated PNG, a broken or oversized image).
+ */
+export async function redactImage({ bytes, mime }, { ocr, firewall, lane = "handoff" }) {
+  const started = performance.now();
+  const buffer = Buffer.from(bytes);
+  const kind = imageKind(buffer);
+  if (!kind) return { refused: `only PNG and JPEG images can be checked for private details, and this one is ${String(mime || "of another type").replace(/^image\//, "").toUpperCase()}` };
+  if (kind === "png" && buffer.includes("acTL")) return { refused: "animated images cannot be checked for private details" };
+  let image;
+  try { image = decodeImage(buffer); } catch (error) { return { refused: `the image could not be read (${error.message})` }; }
+  const read = await ocr.read(image, { lane });
+  const masked = (await sensitiveSpans(read.lines, { firewall, lane })).map((span) => ({ kind: span.kind, box: boxFor(span, image.width, image.height), label: LABELS[span.kind], item: span.item }));
+  if (!masked.length) return { bytes: buffer, mime, masked, ms: Math.round(performance.now() - started) };
+  for (const entry of masked) paint(image, entry.box, entry.label);
+  let opaque = true;
+  for (let p = 3; p < image.data.length && opaque; p += 4) opaque = image.data[p] === 255;
+  const png = pngjs.PNG.sync.write({ width: image.width, height: image.height, data: Buffer.from(image.data.buffer, image.data.byteOffset, image.data.byteLength) }, { colorType: opaque ? 2 : 6, inputHasAlpha: true });
+  return { bytes: png, mime: "image/png", masked, ms: Math.round(performance.now() - started) };
+}

package/lib/runner.mjs

@@ -192,6 +192,8 @@ export function runAgentTask(task, emit, observe = () => {}) {
   let child;
   try {
     child = spawn(invocation.command, invocation.args, { cwd, env: environment.env });
+    // An agent that exits before reading its brief breaks the pipe; its exit says how the run went.
+    child.stdin.on("error", () => {});
     child.stdin.end(invocation.stdin);
   } catch (error) {
     running.delete(slot);

package/lib/runtime-install.mjs

@@ -12,10 +12,10 @@ function dependencyTarget(root, name) {
   return path.join(root, "node_modules", ...name.split("/"));
 }
 
-// What the runtime keeps of a large package. onnxruntime-web ships browser bundles for every
-// backend (125 MB); the daemon needs its Node entry and the one wasm binary that entry loads.
+// What the runtime keeps of a large package. onnxruntime-node ships native binaries for every OS (about 200 MB); the
+// daemon needs its JavaScript and this machine's own binary.
 const KEEP = {
-  "onnxruntime-web": ["package.json", "types.d.ts", "dist/ort.node.min.mjs", "dist/ort.node.min.js", "dist/ort-wasm-simd-threaded.wasm", "dist/ort-wasm-simd-threaded.mjs"],
+  "onnxruntime-node": ["package.json", "dist", "lib", `bin/napi-v6/${process.platform}/${process.arch}`],
 };
 
 function copyPackage(source, target, name) {
@@ -25,7 +25,7 @@ function copyPackage(source, target, name) {
     const from = path.join(source, file);
     if (!fs.existsSync(from)) continue;
     ensureDir(path.dirname(path.join(target, file)));
-    fs.cpSync(from, path.join(target, file), { force: true });
+    fs.cpSync(from, path.join(target, file), { recursive: true, force: true });
   }
 }
 
@@ -76,7 +76,8 @@ export function installRuntime({ sourceRoot, version }) {
       if (!fs.existsSync(source)) continue;
       fs.cpSync(source, path.join(staging, entry), { recursive: true, force: true });
     }
-    copyDependencyClosure(sourceRoot, staging, ["node-sqlite3-wasm", "ws", "onnxruntime-web", "@huggingface/tokenizers", "@clack/prompts", "@clack/core", "picocolors"]);
+    // Everything the package depends on, so a dependency added there cannot be left out here.
+    copyDependencyClosure(sourceRoot, staging, Object.keys(readJson(path.join(sourceRoot, "package.json"), {}).dependencies || {}));
     fs.writeFileSync(path.join(staging, ".ready"), version + "\n", { mode: 0o600 });
     ensureDir(path.dirname(release));
     fs.rmSync(release, { recursive: true, force: true });

package/lib/screens.mjs

@@ -0,0 +1,96 @@
+import { decodeImage } from "./ocr.mjs";
+
+// Two screenshots of the same screen, before and after an agent's UI change: which parts of the picture changed, and
+// what the text on screen says now that it did not before. The pixels find where; the text read on this computer
+// (ocr.mjs) finds what, down to a lost space ("yet OPEN" becoming "yetOPEN"). Code only, no model judges: the agent
+// that made the change reads the list against what it was asked to do.
+
+const CELL = 16;
+// A cell counts as changed when its pixels differ by more than this on average (0 to 255): re-rendering the same page
+// moves anti-aliasing by a little; the ui-checks eval set it against identical renders.
+const CELL_DIFF = 6;
+const MIN_OVERLAP = 0.3;
+
+const luminance = (data, i) => 0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2];
+const overlap = (a, b) => {
+  const w = Math.max(0, Math.min(a[2], b[2]) - Math.max(a[0], b[0])), h = Math.max(0, Math.min(a[3], b[3]) - Math.max(a[1], b[1]));
+  const area = (box) => Math.max(1, (box[2] - box[0]) * (box[3] - box[1]));
+  return (w * h) / Math.min(area(a), area(b));
+};
+
+/** Changed regions of two same-size images: [{ box: [x0, y0, x1, y1], share }], largest first. */
+export function changedRegions(before, after, { cell = CELL, threshold = CELL_DIFF } = {}) {
+  const cols = Math.ceil(before.width / cell), rows = Math.ceil(before.height / cell);
+  const changed = new Uint8Array(cols * rows);
+  for (let cy = 0; cy < rows; cy += 1) for (let cx = 0; cx < cols; cx += 1) {
+    let sum = 0, count = 0;
+    for (let y = cy * cell; y < Math.min(before.height, (cy + 1) * cell); y += 1) for (let x = cx * cell; x < Math.min(before.width, (cx + 1) * cell); x += 1) {
+      const i = (y * before.width + x) * 4;
+      sum += Math.abs(luminance(before.data, i) - luminance(after.data, i)) + Math.abs(before.data[i] - after.data[i]) / 3 + Math.abs(before.data[i + 2] - after.data[i + 2]) / 3;
+      count += 1;
+    }
+    if (sum / count > threshold) changed[cy * cols + cx] = 1;
+  }
+  // Touching changed cells become one region.
+  const regions = [];
+  const seen = new Uint8Array(cols * rows);
+  for (let start = 0; start < changed.length; start += 1) {
+    if (!changed[start] || seen[start]) continue;
+    const stack = [start];
+    seen[start] = 1;
+    let x0 = Infinity, y0 = Infinity, x1 = -1, y1 = -1, cells = 0;
+    while (stack.length) {
+      const at = stack.pop(), cx = at % cols, cy = (at - cx) / cols;
+      cells += 1; x0 = Math.min(x0, cx); y0 = Math.min(y0, cy); x1 = Math.max(x1, cx); y1 = Math.max(y1, cy);
+      for (const [dx, dy] of [[1, 0], [-1, 0], [0, 1], [0, -1], [1, 1], [-1, -1], [1, -1], [-1, 1]]) {
+        const nx = cx + dx, ny = cy + dy, next = ny * cols + nx;
+        if (nx >= 0 && ny >= 0 && nx < cols && ny < rows && changed[next] && !seen[next]) { seen[next] = 1; stack.push(next); }
+      }
+    }
+    regions.push({ box: [x0 * cell, y0 * cell, Math.min(before.width, (x1 + 1) * cell), Math.min(before.height, (y1 + 1) * cell)], share: Math.round((cells / (cols * rows)) * 10_000) / 10_000 });
+  }
+  return regions.sort((a, b) => b.share - a.share);
+}
+
+/** Lines of text read before and after, paired by where they sit: { changed: [{ before, after, box }], moved, removed, added }. */
+export function textChanges(beforeLines, afterLines) {
+  const used = new Set();
+  const changed = [], removed = [];
+  for (const line of beforeLines) {
+    const same = afterLines.findIndex((other, i) => !used.has(i) && other.text === line.text && overlap(line.box, other.box) >= MIN_OVERLAP);
+    if (same >= 0) { used.add(same); continue; }
+    let best = -1, bestOverlap = MIN_OVERLAP;
+    afterLines.forEach((other, i) => { const o = overlap(line.box, other.box); if (!used.has(i) && o >= bestOverlap) { best = i; bestOverlap = o; } });
+    if (best >= 0) { used.add(best); changed.push({ before: line.text, after: afterLines[best].text, box: afterLines[best].box }); }
+    else removed.push({ text: line.text, box: line.box });
+  }
+  const added = afterLines.filter((_, i) => !used.has(i)).map((line) => ({ text: line.text, box: line.box }));
+  // The same words gone from one place and new in another moved.
+  const moved = [];
+  for (const line of [...removed]) {
+    const to = added.findIndex((other) => other.text === line.text);
+    if (to < 0) continue;
+    moved.push({ text: line.text, from: line.box, to: added[to].box });
+    removed.splice(removed.indexOf(line), 1);
+    added.splice(to, 1);
+  }
+  return { changed, moved, removed, added };
+}
+
+/** Compares two screenshots (PNG or JPEG bytes): { size, regions, text, summary }. */
+export async function compareScreens(beforeBytes, afterBytes, { ocr, lane = "interactive" }) {
+  const before = decodeImage(beforeBytes), after = decodeImage(afterBytes);
+  if (!before || !after) throw new Error("both screenshots must be PNG or JPEG");
+  const sameSize = before.width === after.width && before.height === after.height;
+  const regions = sameSize ? changedRegions(before, after) : [];
+  const [readBefore, readAfter] = [await ocr.read(before, { lane }), await ocr.read(after, { lane })];
+  const text = textChanges(readBefore.lines, readAfter.lines);
+  const parts = [];
+  if (!sameSize) parts.push(`the screenshots differ in size (${before.width}x${before.height} and ${after.width}x${after.height}), so only their text was compared`);
+  else parts.push(regions.length ? `${regions.length} region${regions.length === 1 ? "" : "s"} of the picture changed` : "no part of the picture changed");
+  for (const change of text.changed) parts.push(`"${change.before}" now reads "${change.after}"`);
+  for (const line of text.moved) parts.push(`"${line.text}" moved`);
+  if (text.removed.length) parts.push(`gone: ${text.removed.map((line) => `"${line.text}"`).join(", ")}`);
+  if (text.added.length) parts.push(`new: ${text.added.map((line) => `"${line.text}"`).join(", ")}`);
+  return { size: { before: [before.width, before.height], after: [after.width, after.height] }, regions: regions.slice(0, 20), text, summary: `${parts.join("; ")}.`, ms: readBefore.ms + readAfter.ms };
+}

package/lib/sealed.mjs

@@ -13,6 +13,9 @@ import { timingSafeEqual, webcrypto } from "node:crypto";
 // - Every message: AES-256-GCM under HKDF-SHA256(ECDH(device, daemon)), a fresh 96-bit nonce,
 //   and the direction, device and request id bound as associated data. Requests carry a
 //   timestamp; the daemon refuses stale ones and any request id it has already seen.
+// - The other way round, the daemon asks a paired browser (its screen history): the question goes
+//   "d2b" and the answer "b2d" under the same key, so neither opens as a message of the device's
+//   own calls. The browser applies the same freshness and replay rules (phones.mjs ask).
 // Not provided: forward secrecy (both keys are static). Revoking a device ends its key.
 
 const subtle = webcrypto.subtle;

package/lib/service.mjs

@@ -118,12 +118,14 @@ export function installService({ runtimeRoot, nodePath = process.execPath, platf
   if (platform === "darwin") {
     const plist = servicePath();
     ensureDir(path.dirname(plist));
+    // MallocLargeCache=0: macOS otherwise keeps the large blocks ONNX Runtime frees (a screenshot's OCR held 1.6 GB after
+    // it finished; 255 MB with it off, image-redaction eval).
     fs.writeFileSync(plist, `<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0"><dict>
   <key>Label</key><string>${LABEL}</string>
   <key>ProgramArguments</key><array><string>${xml(nodePath)}</string><string>${xml(entry)}</string><string>daemon</string></array>
-  <key>EnvironmentVariables</key><dict><key>PATH</key><string>${xml(envPath)}</string></dict>
+  <key>EnvironmentVariables</key><dict><key>PATH</key><string>${xml(envPath)}</string><key>MallocLargeCache</key><string>0</string></dict>
   <key>RunAtLoad</key><true/><key>KeepAlive</key><true/><key>ThrottleInterval</key><integer>5</integer>
   <key>StandardOutPath</key><string>${xml(paths.log)}</string><key>StandardErrorPath</key><string>${xml(paths.log)}</string>
 </dict></plist>\n`, { mode: 0o600 });
@@ -145,7 +147,7 @@ export function installService({ runtimeRoot, nodePath = process.execPath, platf
   }
   // Other Node platforms get a ready daemon for this login only. `doctor` reports that
   // login persistence is unavailable rather than pretending it was set.
-  const child = spawn(nodePath, [entry, "daemon"], { detached: true, stdio: "ignore", env: { ...process.env, PATH: envPath } });
+  const child = spawn(nodePath, [entry, "daemon"], { detached: true, stdio: "ignore", env: { ...process.env, PATH: envPath, ...(process.platform === "darwin" ? { MallocLargeCache: "0" } : {}) } });
   child.unref();
   return { manager: "detached", file: null };
 }

package/lib/snippets.mjs

@@ -0,0 +1,64 @@
+import fs from "node:fs";
+import path from "node:path";
+import { randomUUID } from "node:crypto";
+import { nameOf } from "./members.mjs";
+import { repositoryOf } from "./worktrees.mjs";
+
+// Code for an agent that does not run on this computer (Grok Bot, Muse) leaves it only with the person's OK. The agent
+// says what it needs to see (request_snippet); the relay finds it in the thread's folder with its local search, takes
+// the lines around it, masks any secret in them, and holds them. The person shares or keeps them (the buddy, or
+// `tryharness board snippet <id> share|keep`); shared lines are posted in the thread for that agent, kept ones never leave.
+
+const BEFORE = 5;
+const LINES = 40;
+
+export function createSnippets({ store, search, scrub, wake = () => {}, notify = () => {}, log = () => {} }) {
+  const read = (id) => { try { return JSON.parse(store.meta(`snippet:${id}`) || "null"); } catch { return null; } };
+  const save = (snippet) => {
+    store.meta(`snippet:${snippet.id}`, JSON.stringify(snippet));
+    const ids = JSON.parse(store.meta("snippets") || "[]");
+    if (!ids.includes(snippet.id)) store.meta("snippets", JSON.stringify([...ids, snippet.id].slice(-200)));
+  };
+
+  /** A cloud agent's request: { ok, snippet } held for the person, or why there is nothing to hold. */
+  const request = async ({ threadId, agent, about }) => {
+    const thread = store.getTask(threadId);
+    const folder = thread?.kind === "goal" ? thread.goal.workspace : null;
+    if (!folder) return { ok: false, error: "this thread has no folder on the person's computer to show code from" };
+    const words = String(about || "").trim().slice(0, 500);
+    if (!words) return { ok: false, error: "say what you need to see" };
+    const found = await search({ query: words, folder, limit: 1 });
+    const file = found?.ok ? found.files[0] : null;
+    if (!file) return { ok: false, error: found?.error || "nothing in this thread's folder matched that" };
+    const root = repositoryOf(folder)?.root ?? folder;
+    let lines;
+    try { lines = fs.readFileSync(path.join(root, file.path), "utf8").split("\n"); } catch (error) { return { ok: false, error: `${file.path} could not be read (${error.code || error.message})` }; }
+    const from = Math.max(1, file.line - BEFORE), to = Math.min(lines.length, from + LINES - 1);
+    // A secret in the lines stays a placeholder for good: it is never filled back in for an agent elsewhere.
+    const { text, masked } = await scrub(lines.slice(from - 1, to).join("\n"));
+    const snippet = { id: randomUUID(), thread: threadId, agent, about: words, path: file.path, from, to, text, masked, state: "waiting", at: new Date().toISOString() };
+    save(snippet);
+    store.roomPost({ goalId: threadId, author: "harness", kind: "note", text: `${nameOf(agent)} asks to see lines ${from} to ${to} of ${file.path} ("${words.slice(0, 120)}"). They stay on this computer until you share them.` });
+    notify(`${nameOf(agent)} asks to see ${to - from + 1} lines of ${path.basename(file.path)}. Share or keep them in Harness.`, snippet);
+    return { ok: true, snippet: { id: snippet.id, path: file.path, from, to, state: "waiting" }, note: "The person decides whether to share these lines; you are told in the thread." };
+  };
+
+  /** The person shares or keeps a waiting snippet. */
+  const decide = (id, share) => {
+    const snippet = read(id);
+    if (!snippet) return { ok: false, error: "no such request" };
+    if (snippet.state !== "waiting") return { ok: false, error: `already ${snippet.state}` };
+    const done = { ...snippet, state: share ? "shared" : "kept", decidedAt: new Date().toISOString() };
+    save(done);
+    if (share) {
+      const post = store.roomPost({ goalId: snippet.thread, author: "harness", kind: "post", mentions: [snippet.agent],
+        text: `@${snippet.agent} the person shared lines ${snippet.from} to ${snippet.to} of ${snippet.path}${snippet.masked ? ` (${snippet.masked} secret${snippet.masked === 1 ? "" : "s"} in them stay masked)` : ""}:\n\n\`\`\`\n${snippet.text}\n\`\`\`` });
+      wake(snippet.thread, post, [snippet.agent]);
+    } else store.roomPost({ goalId: snippet.thread, author: "harness", kind: "note", text: `The person kept lines ${snippet.from} to ${snippet.to} of ${snippet.path} on this computer.` });
+    log(`snippet ${id}: ${done.state}`);
+    return { ok: true, snippet: { id, state: done.state } };
+  };
+
+  const list = ({ all = false } = {}) => JSON.parse(store.meta("snippets") || "[]").map(read).filter((snippet) => snippet && (all || snippet.state === "waiting"));
+  return { request, decide, list };
+}