ravensight-playtest 0.2.8 → 0.3.0

package/src/run/drivers/web.js

@@ -2,6 +2,7 @@ import { mkdir, rename, stat, readdir } from 'node:fs/promises';
 import path from 'node:path';
 import {
   DriverError,
+  STALE_REF_MESSAGE,
   THIN_OBSERVATION_CHARS,
   observationHash,
   screenshotName,
@@ -63,35 +64,337 @@ const INTERACTIVE_ROLES = new Set([
   'canvas'
 ]);
 
+/**
+ * Landmark roles, and how a duplicate's hint names them when they carry no
+ * name of their own.
+ */
+const LANDMARK_LABELS = Object.freeze({
+  banner: 'header',
+  navigation: 'navigation',
+  main: 'main',
+  contentinfo: 'footer',
+  complementary: 'sidebar',
+  region: 'region',
+  dialog: 'dialog',
+  alertdialog: 'dialog',
+  form: 'form',
+  search: 'search',
+  toolbar: 'toolbar',
+  tablist: 'tab list',
+  menu: 'menu',
+  menubar: 'menu bar',
+  group: 'group'
+});
+
+/** Attributes the snapshot prints that tell a persona nothing. */
+const NOISE_ATTRIBUTE = / \[(?:ref=[^\]]*|box=[^\]]*|cursor=[^\]]*|active)\]/g;
+
+/** The `[ref=e12]` Playwright's AI mode snapshot prints. */
+const REF_ATTRIBUTE = /\[ref=(e\d+)\]/;
+
+/** The `[box=x,y,width,height]` a snapshot taken with `boxes` prints. */
+const BOX_ATTRIBUTE = /\[box=(-?[\d.]+),(-?[\d.]+),(-?[\d.]+),(-?[\d.]+)\]/;
+
 /**
  * Keep the nodes a player could act on or read, and drop the structure in
  * between. An aria snapshot prints every wrapper (`- generic:`, `- list:`, a
  * nameless `- img`) and the attribute lines under links (`- /url: ...`), none
  * of which tell a persona anything, and all of which cost tokens every turn.
  *
- * Indentation is dropped too: the persona targets nodes by role and name, not
- * by depth. Text that is not an aria snapshot (the innerText fallback for a
- * thin page) is returned unchanged.
+ * Indentation is dropped too: the persona targets nodes by ref, role and
+ * name, not by depth. Text that is not an aria snapshot (the innerText
+ * fallback for a thin page) is returned unchanged.
  *
  * @param {string} text
+ * @param {{viewport?: {width: number, height: number}}} [options]
  * @returns {string}
  */
-export function compactAriaSnapshot(text) {
+export function compactAriaSnapshot(text, options = {}) {
+  const nodes = compactAriaNodes(text, options);
+  return nodes === null ? String(text == null ? '' : text) : nodes.map(node => node.line).join('\n');
+}
+
+/**
+ * The centre of a box, or null when the node is not visible in the viewport.
+ * Rounded, and bounded by the viewport, so a coordinate the persona copies is
+ * always one the mouse can reach.
+ *
+ * @param {Array<number>|null} box [x, y, width, height]
+ * @param {{width: number, height: number}} viewport
+ * @returns {{x: number, y: number}|null}
+ */
+export function visibleCentre(box, viewport = DEFAULT_VIEWPORT) {
+  if (!box) return null;
+  const [x, y, width, height] = box;
+  if (!(width > 0 && height > 0)) return null;
+  const cx = Math.round(x + width / 2);
+  const cy = Math.round(y + height / 2);
+  if (cx < 0 || cy < 0 || cx > viewport.width || cy > viewport.height) return null;
+  return { x: cx, y: cy };
+}
+
+/**
+ * The kept nodes of an aria snapshot with the depth each one had, or null for
+ * text that is not an aria snapshot. The depth is not shown to the model; it
+ * is what tells repeated siblings apart from a panel that follows them.
+ *
+ * An interactive node from Playwright's AI mode snapshot carries its short ref
+ * (`[e12]`), which `click` and `type` accept as a target, and, when it is on
+ * screen, its centre `@(x,y)`. Two interactive nodes with the same role and
+ * name each get a hint naming the nearest landmark or named container around
+ * them ("in header", "in main"), so the persona can tell them apart.
+ *
+ * `hashLine` is the line without the centre: what "changed" and "new since
+ * the last screen" compare, so a scroll or an animation does not read as a
+ * new screen.
+ *
+ * @param {string} text
+ * @param {{viewport?: {width: number, height: number}}} [options]
+ * @returns {Array<{depth: number, line: string, hashLine: string, body: string, role: string, name: string|null, ref: string|null, centre: Object|null, hint: string|null, key: string|null, duplicate: boolean}>|null}
+ */
+export function compactAriaNodes(text, { viewport = DEFAULT_VIEWPORT } = {}) {
   const value = String(text == null ? '' : text);
   const lines = value.split('\n');
-  if (!lines.some(line => /^\s*- /.test(line))) return value;
+  if (!lines.some(line => /^\s*- /.test(line))) return null;
   const kept = [];
+  const ancestors = [];
   for (const line of lines) {
     const trimmed = line.trim();
     if (!trimmed.startsWith('- ')) continue;
-    const body = unquoteKey(trimmed.slice(2));
-    if (body.startsWith('/')) continue;
-    const role = (body.match(/^([a-z]+)/) || [])[1] || '';
+    const depth = line.length - line.trimStart().length;
+    const raw = unquoteKey(trimmed.slice(2));
+    while (ancestors.length > 0 && ancestors[ancestors.length - 1].depth >= depth) ancestors.pop();
+    if (raw.startsWith('/')) continue;
+    const role = (raw.match(/^([a-z]+)/) || [])[1] || '';
+    const name = (raw.match(/^[a-z]+ "([^"]*)"/) || [])[1] ?? null;
+    const ref = (raw.match(REF_ATTRIBUTE) || [])[1] || null;
+    const boxMatch = raw.match(BOX_ATTRIBUTE);
+    const box = boxMatch ? boxMatch.slice(1, 5).map(Number) : null;
+    const body = raw.replace(NOISE_ATTRIBUTE, '');
+    ancestors.push({ depth, role, name });
     const named = /^[a-z]+ "[^"]*"/.test(body);
     const inlineText = /^[a-z]+(?: "[^"]*")?(?: \[[^\]]*\])*: \S/.test(body);
-    if (INTERACTIVE_ROLES.has(role) || named || inlineText) kept.push(`- ${body}`);
+    const interactive = INTERACTIVE_ROLES.has(role);
+    if (!(interactive || named || inlineText)) continue;
+    const container = ancestors.slice(0, -1).reverse().find(entry => LANDMARK_LABELS[entry.role] || entry.name);
+    const hint = container
+      ? (container.name ? `${container.name}${LANDMARK_LABELS[container.role] && container.role !== 'region' ? ` ${LANDMARK_LABELS[container.role]}` : ''}` : LANDMARK_LABELS[container.role])
+      : null;
+    kept.push({
+      depth,
+      body,
+      role,
+      name,
+      ref: interactive ? ref : null,
+      centre: interactive ? visibleCentre(box, viewport) : null,
+      hint,
+      key: interactive && name ? `${role}|${name}` : null,
+      duplicate: false,
+      line: '',
+      hashLine: ''
+    });
   }
-  return kept.join('\n');
+  const counts = new Map();
+  for (const node of kept) if (node.key) counts.set(node.key, (counts.get(node.key) || 0) + 1);
+  for (const node of kept) {
+    node.duplicate = Boolean(node.key) && counts.get(node.key) > 1;
+    const split = node.body.match(/^([a-z]+(?: "[^"]*")?(?: \[[^\]]*\])*)(.*)$/s) || [null, node.body, ''];
+    const head = split[1];
+    const tail = split[2];
+    const refPart = node.ref ? ` [${node.ref}]` : '';
+    const hintPart = node.duplicate && node.hint ? ` (in ${node.hint})` : '';
+    const centrePart = node.centre ? ` @(${node.centre.x},${node.centre.y})` : '';
+    node.hashLine = `- ${head}${refPart}${hintPart}${tail}`;
+    node.line = `- ${head}${refPart}${centrePart}${hintPart}${tail}`;
+  }
+  return kept;
+}
+
+/** Roles a player types into or sets, kept first when an observation has to be cut. */
+const INPUT_ROLES = new Set(['textbox', 'searchbox', 'combobox', 'checkbox', 'radio', 'slider', 'spinbutton', 'switch', 'listbox']);
+
+/** The share of interactive nodes, counted from the end of the page, kept ahead of the middle when a page is cut. */
+export const TAIL_FRACTION = 0.1;
+
+/** The tail window is never smaller than this many interactive nodes. */
+const TAIL_MIN = 3;
+
+/** A run of this many look-alike siblings or more is collapsed when the page is over the cap. */
+export const COLLAPSE_MIN_RUN = 4;
+
+/** How many of a collapsed run are still shown, so the model sees what the items look like. */
+export const COLLAPSE_KEEP = 2;
+
+/**
+ * The shape of one node, without the words that make it unique: its role,
+ * its state attributes, whether it has a name, whether the name carries
+ * digits and how many separators. Two school cards share a shape; the card
+ * list and a "TAKE THE JOB" button after it do not.
+ *
+ * @param {string} body
+ * @returns {string}
+ */
+function nodeShape(body) {
+  const role = (body.match(/^([a-z]+)/) || [])[1] || '';
+  const name = (body.match(/^[a-z]+ "([^"]*)"/) || [])[1];
+  const attributes = [...body.matchAll(/\[([a-z-]+)(?:=[^\]]*)?\]/g)].map(match => match[1]).filter(key => key !== 'ref').join(',');
+  const nameShape = name === undefined ? '' : `n${/\d/.test(name) ? 'd' : ''}${(name.match(/[·•|/]/g) || []).length}`;
+  const inline = /: \S/.test(body.replace(/^[a-z]+(?: "[^"]*")?/, '')) ? 't' : '';
+  return `${role}|${nameShape}|${attributes}|${inline}`;
+}
+
+/**
+ * Collapse runs of look-alike siblings (a node and its kept descendants,
+ * compared by shape) into the first {@link COLLAPSE_KEEP} plus one line that
+ * says how many more there are.
+ *
+ * @param {Array<{depth: number, line: string, body: string, role: string}>} nodes
+ * @returns {{entries: Array<{line: string, role: string, marker?: boolean}>, collapsed: number}}
+ */
+export function collapseRepeatedSiblings(nodes) {
+  const keyCounts = new Map();
+  for (const node of nodes) if (node.key) keyCounts.set(node.key, (keyCounts.get(node.key) || 0) + 1);
+  const unitEnd = start => {
+    let end = start + 1;
+    while (end < nodes.length && nodes[end].depth > nodes[start].depth) end += 1;
+    return end;
+  };
+  const signature = (start, end) => nodes.slice(start, end).map(node => `${node.depth - nodes[start].depth}:${nodeShape(node.body)}`).join(';');
+  const entries = [];
+  let collapsed = 0;
+  let i = 0;
+  while (i < nodes.length) {
+    const units = [[i, unitEnd(i)]];
+    const sig = signature(i, units[0][1]);
+    let next = units[0][1];
+    while (next < nodes.length && nodes[next].depth === nodes[i].depth) {
+      const end = unitEnd(next);
+      if (signature(next, end) !== sig) break;
+      units.push([next, end]);
+      next = end;
+    }
+    if (units.length >= COLLAPSE_MIN_RUN) {
+      // A node whose role and name also appear outside this run is never
+      // hidden in the count: two buttons called "Run the week", one in the
+      // header and one in the main panel, both stay visible with their refs.
+      const inRun = new Map();
+      for (const node of nodes.slice(i, next)) if (node.key) inRun.set(node.key, (inRun.get(node.key) || 0) + 1);
+      // Only the first in-run twin of a name that also appears outside the
+      // run is kept, beside the first two items. One "View" button in the
+      // header must not stop a list of 249 "View" cards from collapsing.
+      const outside = key => key && (keyCounts.get(key) || 0) > (inRun.get(key) || 0);
+      const shownKeys = new Set();
+      let more = 0;
+      units.forEach((unit, index) => {
+        const unitNodes = nodes.slice(unit[0], unit[1]);
+        const twins = unitNodes.map(node => node.key).filter(key => outside(key) && !shownKeys.has(key));
+        if (index < COLLAPSE_KEEP || twins.length > 0) {
+          for (const node of unitNodes) {
+            entries.push({ line: node.line, hashLine: node.hashLine, role: node.role, key: node.key });
+            if (node.key) shownKeys.add(node.key);
+          }
+        } else {
+          more += 1;
+        }
+      });
+      if (more > 0) {
+        collapsed += more;
+        entries.push({ line: `- ... and ${more} more like it (${nodes[i].role || 'items'}; target one by its name)`, role: '', marker: true });
+      }
+      i = next;
+    } else {
+      entries.push({ line: nodes[i].line, hashLine: nodes[i].hashLine, role: nodes[i].role, key: nodes[i].key });
+      i += 1;
+    }
+  }
+  return { entries, collapsed };
+}
+
+/**
+ * Fit a compacted aria snapshot under the web ceiling without losing the
+ * part of the page the persona has to act on.
+ *
+ * Under the cap nothing changes. Over it, in this order: interactive nodes
+ * are what is kept (the last tenth of them on the page first, since that is
+ * where a form after a long list sits, then inputs, then buttons and links,
+ * then any role with a name); runs of look-alike siblings collapse to their
+ * first two and a count; nodes that are new since the previous observation
+ * come next; then
+ * the rest, and only what still does not fit is trimmed. Kept lines stay in
+ * page order, and a first line says how many items were collapsed and how
+ * many lines were left out, so the model knows the screen was cut.
+ *
+ * @param {string} text the raw aria snapshot
+ * @param {{max?: number, previous?: Set<string>|null}} [options]
+ * @returns {{text: string, lines: Array<string>}|null} null when the text is not an aria snapshot
+ */
+export function fitAriaSnapshot(text, { max = WEB_OBSERVATION_MAX_CHARS, previous = null, viewport = DEFAULT_VIEWPORT } = {}) {
+  const nodes = compactAriaNodes(text, { viewport });
+  if (nodes === null) return null;
+  const lines = nodes.map(node => node.hashLine);
+  const full = nodes.map(node => node.line).join('\n');
+  if (full.length <= max) return { text: full, lines, nodes };
+
+  const { entries, collapsed } = collapseRepeatedSiblings(nodes);
+  const header = (left, out) => {
+    const parts = [];
+    if (left > 0) parts.push(`${left} similar items collapsed`);
+    if (out > 0) parts.push(`${out} lines left out to fit`);
+    return parts.length > 0 ? `[${parts.join('; ')}]` : '';
+  };
+  const collapsedText = entries.map(entry => entry.line).join('\n');
+  const withHeader = `${header(collapsed, 0)}\n${collapsedText}`;
+  if (withHeader.length <= max) return { text: withHeader, lines, nodes };
+
+  const rank = entry => {
+    if (entry.marker) return 0;
+    if (INPUT_ROLES.has(entry.role)) return 1;
+    if (entry.role === 'button' || entry.role === 'link') return 2;
+    if (INTERACTIVE_ROLES.has(entry.role) || /^- [a-z]+ "/.test(entry.line)) return 3;
+    return 5;
+  };
+  const isNew = entry => Boolean(previous) && !entry.marker && !previous.has(entry.hashLine || entry.line);
+  // The last interactive nodes on the page are usually its form or its
+  // confirm button (a panel after a list). When the list could not be
+  // collapsed they would otherwise lose to the list on page order, so the
+  // tail window, the final tenth of interactive nodes, ranks above the middle.
+  const interactive = entries.map((entry, index) => ({ entry, index })).filter(({ entry }) => !entry.marker && rank(entry) < 5);
+  const tailSize = Math.max(TAIL_MIN, Math.ceil(interactive.length * TAIL_FRACTION));
+  const tail = new Set(interactive.slice(-tailSize).map(({ index }) => index));
+  const tierOf = (entry, index) => {
+    if (entry.marker) return 0;
+    if (tail.has(index)) return 1 + (isNew(entry) ? 0 : 0.5);
+    if (rank(entry) < 5) return 2 + rank(entry) * 2 + (isNew(entry) ? 0 : 1);
+    return isNew(entry) ? 20 : 21;
+  };
+  const order = entries
+    .map((entry, index) => ({ entry, index, tier: tierOf(entry, index) }))
+    .sort((a, b) => a.tier - b.tier || a.index - b.index);
+  // Room for the header line, whose figures are not known until the end.
+  const budget = max - 80;
+  const chosen = new Set();
+  let used = 0;
+  for (const candidate of order) {
+    const cost = candidate.entry.line.length + 1;
+    if (used + cost > budget) continue;
+    chosen.add(candidate.index);
+    used += cost;
+  }
+  // Duplicates by name travel together: showing one "Run the week" and not
+  // the other is how a persona ends up clicking a name that matches twice.
+  const chosenKeys = new Set(entries.filter((entry, index) => chosen.has(index) && entry.key).map(entry => entry.key));
+  // Bounded by the cap's own slack: a page whose every card shares a name
+  // is the trim's job, not this one's.
+  entries.forEach((entry, index) => {
+    if (!entry.key || !chosenKeys.has(entry.key) || chosen.has(index)) return;
+    const cost = entry.line.length + 1;
+    if (used + cost > budget) return;
+    chosen.add(index);
+    used += cost;
+  });
+  const kept = entries.filter((entry, index) => chosen.has(index)).map(entry => entry.line);
+  const out = entries.length - kept.length;
+  return { text: truncateObservation(`${header(collapsed, out)}\n${kept.join('\n')}`, max), lines, nodes };
 }
 
 /**
@@ -253,6 +556,78 @@ export function isLoopback(hostname) {
   return LOOPBACK_HOSTS.includes(hostname) || /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(hostname);
 }
 
+/**
+ * A target written as locator code rather than as something on screen. A
+ * persona that writes `getByTestId('strip-advance')` means a selector, and
+ * searching the page for that literal text waits the whole action timeout
+ * for nothing.
+ */
+export const LOCATOR_CODE = /^\s*(?:getBy|page\.|locator\(|css=|xpath=|\/\/)/;
+
+/** What the persona is told when it writes locator code as a target. */
+export const LOCATOR_CODE_MESSAGE = 'targets are refs or visible text, not locator code';
+
+export { STALE_REF_MESSAGE };
+
+/** A ref as the snapshot prints it. */
+export const REF_PATTERN = /^e\d{1,6}$/;
+
+/**
+ * A failure on the driver's side, not the game's: a target that names two
+ * elements, a target that is not a target, a ref that is not on screen. The
+ * transcript records these as `driver_error`, and the report step is told
+ * they are never evidence of a game bug.
+ */
+export class TargetError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'TargetError';
+    this.driverSide = true;
+  }
+}
+
+/**
+ * The candidates a target could mean, from the last snapshot: the
+ * interactive nodes whose name is the target (or whose role and name are,
+ * for `button "Start"`), with their refs and hints.
+ *
+ * @param {Array<Object>|null} nodes
+ * @param {string} target
+ * @returns {Array<Object>}
+ */
+export function candidatesFor(nodes, target) {
+  if (!Array.isArray(nodes)) return [];
+  const value = String(target).trim();
+  const roleAndName = value.match(/^([a-z]+)\s+"([^"]*)"$/);
+  return nodes.filter(node => node.key && node.ref && (roleAndName
+    ? node.role === roleAndName[1] && node.name === roleAndName[2]
+    : String(node.name).trim().toLowerCase() === value.toLowerCase()));
+}
+
+/** The most candidates an ambiguity answer lists before "and N more". */
+export const AMBIGUITY_LIST_MAX = 5;
+
+/**
+ * "2 elements match "Run the week"; use ref e3 (in header) or e5 (in main)".
+ *
+ * @param {string} target
+ * @param {Array<Object>} candidates
+ * @param {number} [count] how many Playwright said matched, when known
+ * @returns {string}
+ */
+export function ambiguityMessage(target, candidates, count = null) {
+  const total = Math.max(count || 0, candidates.length);
+  const shown = candidates.slice(0, AMBIGUITY_LIST_MAX);
+  const listed = shown.map(node => `${node.ref}${node.hint ? ` (in ${node.hint})` : ''}`);
+  const more = candidates.length - shown.length;
+  const choice = more > 0
+    ? `${listed.join(', ')} and ${more} more`
+    : listed.length > 1 ? `${listed.slice(0, -1).join(', ')} or ${listed[listed.length - 1]}` : listed[0];
+  return listed.length > 0
+    ? `${total} elements match ${JSON.stringify(String(target))}; use ref ${choice}`
+    : `${total} elements match ${JSON.stringify(String(target))}; take a snapshot and use the ref of the one you mean`;
+}
+
 /**
  * Turn what the model wrote into a Playwright locator.
  *
@@ -268,6 +643,7 @@ export function isLoopback(hostname) {
  */
 export function resolveLocator(page, target) {
   const value = String(target).trim();
+  if (LOCATOR_CODE.test(value)) throw new TargetError(LOCATOR_CODE_MESSAGE);
   if (KNOWN_SELECTOR_ENGINES.test(value)) return page.locator(value);
 
   // The name group cannot contain a quote (`[^"]*`), so there is nothing to
@@ -281,6 +657,16 @@ export function resolveLocator(page, target) {
   return page.getByText(value, { exact: false }).first();
 }
 
+/**
+ * Give the document a random token. Runs in the browser at the start of every
+ * new document, so a reload or a navigation gets a new one and a single page
+ * app's route change keeps it. Serialised by Playwright, so it must not close
+ * over anything.
+ */
+function markDocument() {
+  globalThis.__ravensightDoc = `${Math.random().toString(36).slice(2)}${Date.now().toString(36)}`;
+}
+
 /**
  * Hand the run's playtest identity to the page. Runs in the browser, before
  * any of the game's own scripts, so the Ravensight JS SDK finds it at
@@ -323,6 +709,17 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
   let shots = 0;
   let lastHash = null;
   let lastObservedHash = null;
+  /** The compacted lines of the last observation, so a cut can prefer what is new. */
+  let lastObservedLines = null;
+  /** The kept nodes of the last observation: refs, names and hints, for a target that matches twice. */
+  let lastNodes = null;
+  let viewportSize = DEFAULT_VIEWPORT;
+  // Refs are numbered per document: after a navigation or a reload, e12 can
+  // be a different element. Each document gets a random token from an init
+  // script; a pushState or hash route change keeps it, a new document does
+  // not. The token the last observation (and so its refs) came from is
+  // compared with the current one before any ref is used.
+  let lastNodesDocument = null;
   const consoleBuffer = [];
 
   /** @returns {any} */
@@ -341,7 +738,17 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
     let text = '';
     let aria = true;
     const body = current.locator('body');
-    if (typeof body.ariaSnapshot === 'function') {
+    // The page level AI mode snapshot carries a ref per element, which
+    // `aria-ref=e12` resolves, and each element's box. Older Playwright
+    // builds have neither, and the body snapshot is the fallback.
+    if (typeof current.ariaSnapshot === 'function') {
+      try {
+        text = await current.ariaSnapshot({ mode: 'ai', boxes: true, timeout: ACTION_TIMEOUT_MS });
+      } catch {
+        text = '';
+      }
+    }
+    if (!text && typeof body.ariaSnapshot === 'function') {
       try {
         text = await body.ariaSnapshot({ timeout: ACTION_TIMEOUT_MS });
       } catch {
@@ -367,13 +774,33 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
   }
 
   /**
-   * What the model is shown for a raw snapshot. `observe` and `act` both use
-   * this, so the two hashes compare like with like.
+   * What the model is shown for a raw snapshot, and the lines it was built
+   * from. A page over the web ceiling is fitted rather than cut at the
+   * character limit (see fitAriaSnapshot), so a panel after a long list
+   * still reaches the model.
    * @param {{text: string, aria: boolean}} snap
-   * @returns {string}
+   * @returns {{text: string, lines: Array<string>|null}}
    */
   function shownText(snap) {
-    return truncateObservation(snap.aria ? compactAriaSnapshot(snap.text) : snap.text, WEB_OBSERVATION_MAX_CHARS);
+    if (snap.aria) {
+      const fitted = fitAriaSnapshot(snap.text, { max: WEB_OBSERVATION_MAX_CHARS, previous: lastObservedLines ? new Set(lastObservedLines) : null, viewport: viewportSize });
+      if (fitted) return fitted;
+    }
+    return { text: truncateObservation(snap.text, WEB_OBSERVATION_MAX_CHARS), lines: null, nodes: null };
+  }
+
+  /**
+   * What `changed` and `unchanged` compare. For an aria snapshot that is the
+   * whole compacted tree, not the fitted text: the fit depends on the
+   * previous observation, so hashing it would read the same screen as
+   * changed. For the innerText fallback it is the truncated text, so a
+   * difference the model never saw is not a change.
+   * @param {{text: string, aria: boolean}} snap
+   * @param {{text: string, lines: Array<string>|null}} shown
+   * @returns {string}
+   */
+  function hashText(snap, shown) {
+    return shown.lines ? shown.lines.join('\n') : shown.text;
   }
 
   /**
@@ -391,7 +818,7 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
    */
   async function typeText(current, action) {
     const text = String(action.text == null ? '' : action.text);
-    const field = action.target ? resolveLocator(current, action.target) : null;
+    const field = action.target || action.ref ? await targetFor(current, action) : null;
     if (action.replace) {
       try {
         await (field || current.locator('*:focus')).fill(text, { timeout: field ? ACTION_TIMEOUT_MS : FOCUSED_FILL_TIMEOUT_MS });
@@ -409,8 +836,75 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
     if (text) await current.keyboard.type(text, { delay: 20 });
   }
 
+  /**
+   * The locator a click or a typing target means. A ref is preferred and
+   * checked against the page; locator code is refused at once, with no wait;
+   * a name that matches more than one interactive node on the last screen is
+   * answered with the candidates and their refs rather than guessed.
+   *
+   * @param {any} current
+   * @param {{ref?: string, target?: string}} action
+   */
+  async function targetFor(current, action) {
+    if (action.ref !== undefined && action.ref !== null && action.ref !== '') {
+      const ref = String(action.ref).trim();
+      if (!REF_PATTERN.test(ref)) throw new TargetError(`a ref looks like e12, as the snapshot prints it; ${JSON.stringify(ref)} is not one`);
+      await assertSameDocument(current);
+      const locator = current.locator(`aria-ref=${ref}`);
+      if (typeof locator.count === 'function' && (await locator.count()) === 0) {
+        throw new TargetError(`ref ${ref} is not on the current screen; take a snapshot and use a ref from it`);
+      }
+      return locator;
+    }
+    const target = String(action.target == null ? '' : action.target);
+    const candidates = candidatesFor(lastNodes, target);
+    if (candidates.length > 1) throw new TargetError(ambiguityMessage(target, candidates));
+    if (LOCATOR_CODE.test(target)) {
+      // Text that only looks like code: a node on screen really is named
+      // that, so it is clicked by its ref rather than refused.
+      if (candidates.length === 1) {
+        await assertSameDocument(current);
+        return current.locator(`aria-ref=${candidates[0].ref}`);
+      }
+      throw new TargetError(LOCATOR_CODE_MESSAGE);
+    }
+    return resolveLocator(current, target);
+  }
+
+  /**
+   * The current document's token, read from the main frame only (a child
+   * frame has its own document and its own token), or null when the page
+   * cannot answer.
+   */
+  async function documentToken(current) {
+    try {
+      const token = await current.evaluate(() => globalThis.__ravensightDoc || null);
+      return typeof token === 'string' && token ? token : null;
+    } catch {
+      return null;
+    }
+  }
+
+  /** Refuse a ref when the document changed since the observation it came from. */
+  async function assertSameDocument(current) {
+    const now = await documentToken(current);
+    if (lastNodesDocument && now !== lastNodesDocument) throw new TargetError(STALE_REF_MESSAGE);
+  }
+
+  /** The hash of the screen as it is now, on observe()'s basis. */
+  async function currentHash(current) {
+    const snap = await snapshotText();
+    const after = { text: hashText(snap, shownText(snap)) };
+    if (String(snap.text).trim().length < THIN_OBSERVATION_CHARS) {
+      const frame = await current.screenshot({ type: 'png', timeout: ACTION_TIMEOUT_MS });
+      after.screenshotBase64 = frame.toString('base64');
+    }
+    return observationHash(after);
+  }
+
   return {
     name: 'playwright_web',
+    supportsRefs: true,
     editModifier,
     keepObservations: WEB_KEEP_OBSERVATIONS,
 
@@ -424,6 +918,7 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
     async launch({ target: url, runDir, recordVideo = false, viewport = DEFAULT_VIEWPORT }) {
       if (!url) throw new DriverError('playwright_web needs a target URL', { fatal: true });
       target = url;
+      viewportSize = viewport || DEFAULT_VIEWPORT;
       runDirectory = runDir;
       videoRequested = Boolean(recordVideo);
 
@@ -447,6 +942,9 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
           persona: playtest.persona || null
         });
       }
+      // One token per document: set again by every new document, kept by
+      // pushState and hash route changes, which do not run init scripts.
+      if (typeof context.addInitScript === 'function') await context.addInitScript(markDocument);
       page = await context.newPage();
       page.on('console', message => {
         if (message.type() === 'error') consoleBuffer.push({ type: 'console', text: message.text() });
@@ -468,9 +966,10 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
       step += 1;
       const snap = await snapshotText();
       const text = snap.text;
+      const shown = shownText(snap);
       const observation = {
         step,
-        text: shownText(snap),
+        text: shown.text,
         url: current.url(),
         elapsedMs: Date.now() - started
       };
@@ -497,11 +996,13 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
           // read it as a blank screen, which is what it is.
         }
       }
-      // Hash what was REPORTED, not the untruncated snapshot: `act` compares
-      // against this, so two screens that differ only past the truncation
-      // point would otherwise read as changed to `act` and identical to
-      // anything reading the observation.
-      lastHash = observationHash(observation);
+      // Hash what hashText says, which `act` hashes the same way: the whole
+      // compacted tree for an aria snapshot (a change anywhere in it is shown,
+      // because a fit keeps new nodes first), the truncated text otherwise.
+      lastHash = observationHash({ text: hashText(snap, shown), screenshotBase64: observation.screenshotBase64 });
+      lastObservedLines = shown.lines;
+      lastNodes = shown.nodes || null;
+      lastNodesDocument = await documentToken(current);
       // The same screen twice is sent as a marker rather than in full
       // (./tools.js), which is most of what a stuck persona would otherwise
       // pay for.
@@ -527,11 +1028,11 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
             if (action.submit) await current.keyboard.press('Enter');
             break;
           case 'click': {
-            if (action.target && typeof action.target === 'object') {
+            if (!action.ref && action.target && typeof action.target === 'object') {
               await current.mouse.click(Number(action.target.x), Number(action.target.y));
               break;
             }
-            await resolveLocator(current, action.target).click({ timeout: ACTION_TIMEOUT_MS });
+            await (await targetFor(current, action)).click({ timeout: ACTION_TIMEOUT_MS });
             break;
           }
           case 'navigate': {
@@ -549,20 +1050,44 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
             return { ok: false, changed: false, error: `the playwright_web driver cannot do "${action.kind}"` };
         }
       } catch (error) {
-        return { ok: false, changed: false, error: String(error && error.message ? error.message : error) };
+        // Playwright colours its call log; the colour codes are noise here.
+        const text = String(error && error.message ? error.message : error).replace(/\u001b\[[0-9;]*m/g, '');
+        // Two elements under one name: Playwright's strict mode. Answered
+        // with the candidates and their refs, so the next try can work.
+        if (/strict mode violation/i.test(text)) {
+          const count = Number((text.match(/resolved to (\d+) elements/) || [])[1]) || null;
+          const candidates = action.target && typeof action.target === 'string' ? candidatesFor(lastNodes, action.target) : [];
+          return { ok: false, changed: false, error: ambiguityMessage(action.target || action.ref || '', candidates, count), driverError: true };
+        }
+        if (error && error.driverSide) return { ok: false, changed: false, error: text, driverError: true };
+        // A click the page kept intercepting is the game's: something sat on
+        // top of the target and never went away. Named, never a driver error.
+        const interceptor = text.match(/^\s*-?\s*(.+?)\s+(?:from .+? subtree\s+)?intercepts pointer events/m);
+        const intercepted = interceptor ? interceptor[1] : null;
+        if (/Timeout \d+ms exceeded/i.test(text) && intercepted) {
+          return { ok: false, changed: false, error: `timed out: ${intercepted.slice(0, 160)} intercepts pointer events over the target`, driverError: false };
+        }
+        // A timeout with the page unchanged while resolving a click or typing
+        // target means the target was never found or never actionable: the
+        // driver's failure, not the game's. Never for navigate: a game that
+        // never loads is a real blocker.
+        if (/Timeout \d+ms exceeded/i.test(text) && (action.kind === 'click' || action.kind === 'type')) {
+          let unchanged = false;
+          try {
+            unchanged = (await currentHash(current)) === before;
+          } catch {
+            unchanged = false;
+          }
+          return { ok: false, changed: false, error: text, driverError: unchanged };
+        }
+        return { ok: false, changed: false, error: text };
       }
 
       let changed = false;
       try {
-        // The same truncation observe() applies, so the two hashes are
-        // comparable. Without it a long screen is always "changed".
-        const snap = await snapshotText();
-        const after = { text: shownText(snap) };
-        if (String(snap.text).trim().length < THIN_OBSERVATION_CHARS) {
-          const frame = await current.screenshot({ type: 'png', timeout: ACTION_TIMEOUT_MS });
-          after.screenshotBase64 = frame.toString('base64');
-        }
-        changed = observationHash(after) !== before;
+        // The same basis observe() hashes, so the two are comparable.
+        // Without it a long screen is always "changed".
+        changed = (await currentHash(current)) !== before;
       } catch {
         changed = true;
       }