tryharness 0.7.22 → 0.7.25

package/lib/screens.mjs

@@ -0,0 +1,212 @@
+import { decodeImage } from "./ocr.mjs";
+
+// Two screenshots of the same screen, before and after an agent's UI change: which parts of the picture changed, and
+// what the text on screen says now that it did not before. The pixels find where; the text read on this computer
+// (ocr.mjs) finds what, down to a lost space ("yet OPEN" becoming "yetOPEN"). Code only, no model judges: the agent
+// that made the change reads the list against what it was asked to do.
+
+const CELL = 16;
+// A cell counts as changed when its pixels differ by more than this on average (0 to 255): re-rendering the same page
+// moves anti-aliasing by a little; the ui-checks eval set it against identical renders.
+const CELL_DIFF = 6;
+const MIN_OVERLAP = 0.3;
+
+const luminance = (data, i) =>
+  0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2];
+const overlap = (a, b) => {
+  const w = Math.max(0, Math.min(a[2], b[2]) - Math.max(a[0], b[0])),
+    h = Math.max(0, Math.min(a[3], b[3]) - Math.max(a[1], b[1]));
+  const area = (box) => Math.max(1, (box[2] - box[0]) * (box[3] - box[1]));
+  return (w * h) / Math.min(area(a), area(b));
+};
+
+/** Changed regions of two same-size images: [{ box: [x0, y0, x1, y1], share }], largest first. */
+export function changedRegions(
+  before,
+  after,
+  { cell = CELL, threshold = CELL_DIFF } = {}
+) {
+  const cols = Math.ceil(before.width / cell),
+    rows = Math.ceil(before.height / cell);
+  const changed = new Uint8Array(cols * rows);
+  for (let cy = 0; cy < rows; cy += 1)
+    for (let cx = 0; cx < cols; cx += 1) {
+      let sum = 0,
+        count = 0;
+      for (
+        let y = cy * cell;
+        y < Math.min(before.height, (cy + 1) * cell);
+        y += 1
+      )
+        for (
+          let x = cx * cell;
+          x < Math.min(before.width, (cx + 1) * cell);
+          x += 1
+        ) {
+          const i = (y * before.width + x) * 4;
+          sum +=
+            Math.abs(luminance(before.data, i) - luminance(after.data, i)) +
+            Math.abs(before.data[i] - after.data[i]) / 3 +
+            Math.abs(before.data[i + 2] - after.data[i + 2]) / 3;
+          count += 1;
+        }
+      if (sum / count > threshold) changed[cy * cols + cx] = 1;
+    }
+  // Touching changed cells become one region.
+  const regions = [];
+  const seen = new Uint8Array(cols * rows);
+  for (let start = 0; start < changed.length; start += 1) {
+    if (!changed[start] || seen[start]) continue;
+    const stack = [start];
+    seen[start] = 1;
+    let x0 = Infinity,
+      y0 = Infinity,
+      x1 = -1,
+      y1 = -1,
+      cells = 0;
+    while (stack.length) {
+      const at = stack.pop(),
+        cx = at % cols,
+        cy = (at - cx) / cols;
+      cells += 1;
+      x0 = Math.min(x0, cx);
+      y0 = Math.min(y0, cy);
+      x1 = Math.max(x1, cx);
+      y1 = Math.max(y1, cy);
+      for (const [dx, dy] of [
+        [1, 0],
+        [-1, 0],
+        [0, 1],
+        [0, -1],
+        [1, 1],
+        [-1, -1],
+        [1, -1],
+        [-1, 1],
+      ]) {
+        const nx = cx + dx,
+          ny = cy + dy,
+          next = ny * cols + nx;
+        if (
+          nx >= 0 &&
+          ny >= 0 &&
+          nx < cols &&
+          ny < rows &&
+          changed[next] &&
+          !seen[next]
+        ) {
+          seen[next] = 1;
+          stack.push(next);
+        }
+      }
+    }
+    regions.push({
+      box: [
+        x0 * cell,
+        y0 * cell,
+        Math.min(before.width, (x1 + 1) * cell),
+        Math.min(before.height, (y1 + 1) * cell),
+      ],
+      share: Math.round((cells / (cols * rows)) * 10_000) / 10_000,
+    });
+  }
+  return regions.sort((a, b) => b.share - a.share);
+}
+
+/** Lines of text read before and after, paired by where they sit: { changed: [{ before, after, box }], moved, removed, added }. */
+export function textChanges(beforeLines, afterLines) {
+  const used = new Set();
+  const changed = [],
+    removed = [];
+  for (const line of beforeLines) {
+    const same = afterLines.findIndex(
+      (other, i) =>
+        !used.has(i) &&
+        other.text === line.text &&
+        overlap(line.box, other.box) >= MIN_OVERLAP
+    );
+    if (same >= 0) {
+      used.add(same);
+      continue;
+    }
+    let best = -1,
+      bestOverlap = MIN_OVERLAP;
+    afterLines.forEach((other, i) => {
+      const o = overlap(line.box, other.box);
+      if (!used.has(i) && o >= bestOverlap) {
+        best = i;
+        bestOverlap = o;
+      }
+    });
+    if (best >= 0) {
+      used.add(best);
+      changed.push({
+        before: line.text,
+        after: afterLines[best].text,
+        box: afterLines[best].box,
+      });
+    } else removed.push({ text: line.text, box: line.box });
+  }
+  const added = afterLines
+    .filter((_, i) => !used.has(i))
+    .map((line) => ({ text: line.text, box: line.box }));
+  // The same words gone from one place and new in another moved.
+  const moved = [];
+  for (const line of [...removed]) {
+    const to = added.findIndex((other) => other.text === line.text);
+    if (to < 0) continue;
+    moved.push({ text: line.text, from: line.box, to: added[to].box });
+    removed.splice(removed.indexOf(line), 1);
+    added.splice(to, 1);
+  }
+  return { changed, moved, removed, added };
+}
+
+/** Compares two screenshots (PNG or JPEG bytes): { size, regions, text, summary }. */
+export async function compareScreens(
+  beforeBytes,
+  afterBytes,
+  { ocr, lane = "interactive" }
+) {
+  const before = decodeImage(beforeBytes),
+    after = decodeImage(afterBytes);
+  if (!before || !after)
+    throw new Error("both screenshots must be PNG or JPEG");
+  const sameSize =
+    before.width === after.width && before.height === after.height;
+  const regions = sameSize ? changedRegions(before, after) : [];
+  const [readBefore, readAfter] = [
+    await ocr.read(before, { lane }),
+    await ocr.read(after, { lane }),
+  ];
+  const text = textChanges(readBefore.lines, readAfter.lines);
+  const parts = [];
+  if (!sameSize)
+    parts.push(
+      `the screenshots differ in size (${before.width}x${before.height} and ${after.width}x${after.height}), so only their text was compared`
+    );
+  else
+    parts.push(
+      regions.length
+        ? `${regions.length} region${regions.length === 1 ? "" : "s"} of the picture changed`
+        : "no part of the picture changed"
+    );
+  for (const change of text.changed)
+    parts.push(`"${change.before}" now reads "${change.after}"`);
+  for (const line of text.moved) parts.push(`"${line.text}" moved`);
+  if (text.removed.length)
+    parts.push(
+      `gone: ${text.removed.map((line) => `"${line.text}"`).join(", ")}`
+    );
+  if (text.added.length)
+    parts.push(`new: ${text.added.map((line) => `"${line.text}"`).join(", ")}`);
+  return {
+    size: {
+      before: [before.width, before.height],
+      after: [after.width, after.height],
+    },
+    regions: regions.slice(0, 20),
+    text,
+    summary: `${parts.join("; ")}.`,
+    ms: readBefore.ms + readAfter.ms,
+  };
+}

package/lib/sealed.mjs

@@ -13,6 +13,9 @@ import { timingSafeEqual, webcrypto } from "node:crypto";
 // - Every message: AES-256-GCM under HKDF-SHA256(ECDH(device, daemon)), a fresh 96-bit nonce,
 //   and the direction, device and request id bound as associated data. Requests carry a
 //   timestamp; the daemon refuses stale ones and any request id it has already seen.
+// - The other way round, the daemon asks a paired browser (its screen history): the question goes
+//   "d2b" and the answer "b2d" under the same key, so neither opens as a message of the device's
+//   own calls. The browser applies the same freshness and replay rules (phones.mjs ask).
 // Not provided: forward secrecy (both keys are static). Revoking a device ends its key.
 
 const subtle = webcrypto.subtle;
@@ -23,57 +26,135 @@ export const SEALED_VERSION = 1;
 export const MAX_SKEW_MS = 5 * 60_000;
 
 export const b64u = (bytes) => Buffer.from(bytes).toString("base64url");
-export const fromB64u = (text) => new Uint8Array(Buffer.from(String(text), "base64url"));
+export const fromB64u = (text) =>
+  new Uint8Array(Buffer.from(String(text), "base64url"));
 
 export async function loadOrCreateIdentity(file) {
   try {
     const saved = JSON.parse(fs.readFileSync(file, "utf8"));
-    const privateKey = await subtle.importKey("jwk", saved.privateJwk, CURVE, false, ["deriveBits"]);
+    const privateKey = await subtle.importKey(
+      "jwk",
+      saved.privateJwk,
+      CURVE,
+      false,
+      ["deriveBits"]
+    );
     return { privateKey, publicKey: saved.publicKey };
   } catch (error) {
-    if (error.code !== "ENOENT") throw new Error(`the phone identity at ${file} is unreadable: ${error.message}`);
+    if (error.code !== "ENOENT")
+      throw new Error(
+        `the phone identity at ${file} is unreadable: ${error.message}`
+      );
   }
   const pair = await subtle.generateKey(CURVE, true, ["deriveBits"]);
-  const publicKey = b64u(new Uint8Array(await subtle.exportKey("raw", pair.publicKey)));
+  const publicKey = b64u(
+    new Uint8Array(await subtle.exportKey("raw", pair.publicKey))
+  );
   const privateJwk = await subtle.exportKey("jwk", pair.privateKey);
-  fs.writeFileSync(file, JSON.stringify({ version: SEALED_VERSION, publicKey, privateJwk }), { mode: 0o600, flag: "wx" });
+  fs.writeFileSync(
+    file,
+    JSON.stringify({ version: SEALED_VERSION, publicKey, privateJwk }),
+    { mode: 0o600, flag: "wx" }
+  );
   return loadOrCreateIdentity(file);
 }
 
-export async function pairProof(secret, offerId, devicePublicKey, daemonPublicKey) {
-  const key = await subtle.importKey("raw", fromB64u(secret), { name: "HMAC", hash: "SHA-256" }, false, ["sign"]);
-  return b64u(new Uint8Array(await subtle.sign("HMAC", key, encoder.encode(`HSC1-pair\n${offerId}\n${devicePublicKey}\n${daemonPublicKey}`))));
+export async function pairProof(
+  secret,
+  offerId,
+  devicePublicKey,
+  daemonPublicKey
+) {
+  const key = await subtle.importKey(
+    "raw",
+    fromB64u(secret),
+    { name: "HMAC", hash: "SHA-256" },
+    false,
+    ["sign"]
+  );
+  return b64u(
+    new Uint8Array(
+      await subtle.sign(
+        "HMAC",
+        key,
+        encoder.encode(
+          `HSC1-pair\n${offerId}\n${devicePublicKey}\n${daemonPublicKey}`
+        )
+      )
+    )
+  );
 }
 
 export async function channelKey(privateKey, peerPublicKey, deviceId) {
-  const peer = await subtle.importKey("raw", fromB64u(peerPublicKey), CURVE, false, []);
-  const shared = await subtle.deriveBits({ name: "ECDH", public: peer }, privateKey, 256);
-  const material = await subtle.importKey("raw", shared, "HKDF", false, ["deriveKey"]);
+  const peer = await subtle.importKey(
+    "raw",
+    fromB64u(peerPublicKey),
+    CURVE,
+    false,
+    []
+  );
+  const shared = await subtle.deriveBits(
+    { name: "ECDH", public: peer },
+    privateKey,
+    256
+  );
+  const material = await subtle.importKey("raw", shared, "HKDF", false, [
+    "deriveKey",
+  ]);
   return subtle.deriveKey(
-    { name: "HKDF", hash: "SHA-256", salt: encoder.encode("HSC1"), info: encoder.encode(`HSC1 channel ${deviceId}`) },
-    material, { name: "AES-GCM", length: 256 }, false, ["encrypt", "decrypt"],
+    {
+      name: "HKDF",
+      hash: "SHA-256",
+      salt: encoder.encode("HSC1"),
+      info: encoder.encode(`HSC1 channel ${deviceId}`),
+    },
+    material,
+    { name: "AES-GCM", length: 256 },
+    false,
+    ["encrypt", "decrypt"]
   );
 }
 
-const aad = (direction, deviceId, requestId) => encoder.encode(`HSC1|${direction}|${deviceId}|${requestId}`);
+const aad = (direction, deviceId, requestId) =>
+  encoder.encode(`HSC1|${direction}|${deviceId}|${requestId}`);
 
 export async function seal(key, direction, deviceId, requestId, value) {
   const iv = webcrypto.getRandomValues(new Uint8Array(12));
-  const ct = await subtle.encrypt({ name: "AES-GCM", iv, additionalData: aad(direction, deviceId, requestId) }, key, encoder.encode(JSON.stringify(value)));
-  return { v: SEALED_VERSION, deviceId, requestId, iv: b64u(iv), ct: b64u(new Uint8Array(ct)) };
+  const ct = await subtle.encrypt(
+    {
+      name: "AES-GCM",
+      iv,
+      additionalData: aad(direction, deviceId, requestId),
+    },
+    key,
+    encoder.encode(JSON.stringify(value))
+  );
+  return {
+    v: SEALED_VERSION,
+    deviceId,
+    requestId,
+    iv: b64u(iv),
+    ct: b64u(new Uint8Array(ct)),
+  };
 }
 
 // Throws on a wrong key, a wrong direction, device or request id, or any changed byte.
 export async function open(key, direction, envelope) {
   const plain = await subtle.decrypt(
-    { name: "AES-GCM", iv: fromB64u(envelope.iv), additionalData: aad(direction, envelope.deviceId, envelope.requestId) },
-    key, fromB64u(envelope.ct),
+    {
+      name: "AES-GCM",
+      iv: fromB64u(envelope.iv),
+      additionalData: aad(direction, envelope.deviceId, envelope.requestId),
+    },
+    key,
+    fromB64u(envelope.ct)
   );
   return JSON.parse(decoder.decode(plain));
 }
 
 export function constantTimeEqual(a, b) {
-  const left = Buffer.from(String(a)), right = Buffer.from(String(b));
+  const left = Buffer.from(String(a)),
+    right = Buffer.from(String(b));
   return left.length === right.length && timingSafeEqual(left, right);
 }
 
@@ -82,31 +163,82 @@ export function constantTimeEqual(a, b) {
 // associated data. A peer's key comes from Harness's directory the first time and is pinned by
 // handle on this machine; a different key later is refused until the person trusts it.
 export async function relayKey(privateKey, peerPublicKey) {
-  const peer = await subtle.importKey("raw", fromB64u(peerPublicKey), CURVE, false, []);
-  const shared = await subtle.deriveBits({ name: "ECDH", public: peer }, privateKey, 256);
-  const material = await subtle.importKey("raw", shared, "HKDF", false, ["deriveKey"]);
-  return subtle.deriveKey({ name: "HKDF", hash: "SHA-256", salt: encoder.encode("HSC1"), info: encoder.encode("HSC1 relay") }, material, { name: "AES-GCM", length: 256 }, false, ["encrypt", "decrypt"]);
+  const peer = await subtle.importKey(
+    "raw",
+    fromB64u(peerPublicKey),
+    CURVE,
+    false,
+    []
+  );
+  const shared = await subtle.deriveBits(
+    { name: "ECDH", public: peer },
+    privateKey,
+    256
+  );
+  const material = await subtle.importKey("raw", shared, "HKDF", false, [
+    "deriveKey",
+  ]);
+  return subtle.deriveKey(
+    {
+      name: "HKDF",
+      hash: "SHA-256",
+      salt: encoder.encode("HSC1"),
+      info: encoder.encode("HSC1 relay"),
+    },
+    material,
+    { name: "AES-GCM", length: 256 },
+    false,
+    ["encrypt", "decrypt"]
+  );
 }
 export async function sealBound(key, bound, value) {
   const iv = webcrypto.getRandomValues(new Uint8Array(12));
-  const ct = await subtle.encrypt({ name: "AES-GCM", iv, additionalData: encoder.encode(`HSC1|${bound}`) }, key, encoder.encode(JSON.stringify(value)));
+  const ct = await subtle.encrypt(
+    { name: "AES-GCM", iv, additionalData: encoder.encode(`HSC1|${bound}`) },
+    key,
+    encoder.encode(JSON.stringify(value))
+  );
   return { iv: b64u(iv), ct: b64u(new Uint8Array(ct)) };
 }
 export async function openBound(key, bound, sealed) {
-  const plain = await subtle.decrypt({ name: "AES-GCM", iv: fromB64u(sealed.iv), additionalData: encoder.encode(`HSC1|${bound}`) }, key, fromB64u(sealed.ct));
+  const plain = await subtle.decrypt(
+    {
+      name: "AES-GCM",
+      iv: fromB64u(sealed.iv),
+      additionalData: encoder.encode(`HSC1|${bound}`),
+    },
+    key,
+    fromB64u(sealed.ct)
+  );
   return JSON.parse(decoder.decode(plain));
 }
 // A file's bytes, sealed as they are (no JSON or base64 inside): bound to its thread and file id like a message.
 export async function sealBytes(key, bound, bytes) {
   const iv = webcrypto.getRandomValues(new Uint8Array(12));
-  const ct = await subtle.encrypt({ name: "AES-GCM", iv, additionalData: encoder.encode(`HSC1|${bound}`) }, key, bytes);
+  const ct = await subtle.encrypt(
+    { name: "AES-GCM", iv, additionalData: encoder.encode(`HSC1|${bound}`) },
+    key,
+    bytes
+  );
   return { iv: b64u(iv), ct: b64u(new Uint8Array(ct)) };
 }
 export async function openBytes(key, bound, sealed) {
-  return Buffer.from(await subtle.decrypt({ name: "AES-GCM", iv: fromB64u(sealed.iv), additionalData: encoder.encode(`HSC1|${bound}`) }, key, fromB64u(sealed.ct)));
+  return Buffer.from(
+    await subtle.decrypt(
+      {
+        name: "AES-GCM",
+        iv: fromB64u(sealed.iv),
+        additionalData: encoder.encode(`HSC1|${bound}`),
+      },
+      key,
+      fromB64u(sealed.ct)
+    )
+  );
 }
 // What two people read to each other to check they hold each other's key: SHA-256 of the key, 20 hex digits in groups of four.
 export async function fingerprint(publicKey) {
-  const digest = Buffer.from(await subtle.digest("SHA-256", fromB64u(publicKey))).toString("hex");
+  const digest = Buffer.from(
+    await subtle.digest("SHA-256", fromB64u(publicKey))
+  ).toString("hex");
   return digest.slice(0, 20).match(/.{4}/g).join(" ");
 }

package/lib/secrets.mjs

@@ -8,18 +8,31 @@ const PATTERNS = [
   { reason: "a private key block", test: /-----BEGIN [A-Z ]*PRIVATE KEY-----/ },
   { reason: "an OpenAI-style API key", test: /\bsk-[A-Za-z0-9_-]{20,}\b/ },
   { reason: "an AWS access key", test: /\bAKIA[0-9A-Z]{16}\b/ },
-  { reason: "a GitHub token", test: /\b(ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{30,}\b|\bgithub_pat_[A-Za-z0-9_]{40,}\b/ },
+  {
+    reason: "a GitHub token",
+    test: /\b(ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{30,}\b|\bgithub_pat_[A-Za-z0-9_]{40,}\b/,
+  },
   { reason: "a Slack token", test: /\bxox[abpr]-[A-Za-z0-9-]{10,}\b/ },
-  { reason: "a JSON web token", test: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/ },
+  {
+    reason: "a JSON web token",
+    test: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/,
+  },
   { reason: "a 32-byte hex key", test: /\b(0x)?[0-9a-fA-F]{64}\b/ },
   { reason: "a Harness connection token", test: /\bhmcp_[A-Za-z0-9]{12,}\b/ },
-  { reason: "a secret assigned in env style", test: /\b[A-Z0-9_]*(SECRET|TOKEN|PASSWORD|PRIVATE_KEY|API_KEY)[A-Z0-9_]*\s*[=:]\s*['"]?[^\s'"]{8,}/ },
-  { reason: "a seed phrase", test: /\b(seed phrase|mnemonic|recovery phrase)\b[\s\S]{0,40}?(\b[a-z]{3,8}\b[\s,]+){11,}/i },
+  {
+    reason: "a secret assigned in env style",
+    test: /\b[A-Z0-9_]*(SECRET|TOKEN|PASSWORD|PRIVATE_KEY|API_KEY)[A-Z0-9_]*\s*[=:]\s*['"]?[^\s'"]{8,}/,
+  },
+  {
+    reason: "a seed phrase",
+    test: /\b(seed phrase|mnemonic|recovery phrase)\b[\s\S]{0,40}?(\b[a-z]{3,8}\b[\s,]+){11,}/i,
+  },
 ];
 
 /** The reason a text must be held, or null when nothing in it looks like a secret. */
 export function secretReason(text) {
   const value = String(text || "");
-  for (const pattern of PATTERNS) if (pattern.test.test(value)) return pattern.reason;
+  for (const pattern of PATTERNS)
+    if (pattern.test.test(value)) return pattern.reason;
   return null;
 }