ravensight-playtest 0.2.9 → 0.3.1

package/src/run/drivers/web.js

@@ -2,6 +2,7 @@ import { mkdir, rename, stat, readdir } from 'node:fs/promises';
 import path from 'node:path';
 import {
   DriverError,
+  STALE_REF_MESSAGE,
   THIN_OBSERVATION_CHARS,
   observationHash,
   screenshotName,
@@ -63,48 +64,289 @@ const INTERACTIVE_ROLES = new Set([
   'canvas'
 ]);
 
+/**
+ * Landmark roles, and how a duplicate's hint names them when they carry no
+ * name of their own.
+ */
+const LANDMARK_LABELS = Object.freeze({
+  banner: 'header',
+  navigation: 'navigation',
+  main: 'main',
+  contentinfo: 'footer',
+  complementary: 'sidebar',
+  region: 'region',
+  dialog: 'dialog',
+  alertdialog: 'dialog',
+  form: 'form',
+  search: 'search',
+  toolbar: 'toolbar',
+  tablist: 'tab list',
+  menu: 'menu',
+  menubar: 'menu bar',
+  group: 'group'
+});
+
+/** Attributes the snapshot prints that tell a persona nothing. */
+const NOISE_ATTRIBUTE = / \[(?:ref=[^\]]*|box=[^\]]*|cursor=[^\]]*|active)\]/g;
+
+/** The `[ref=e12]` Playwright's AI mode snapshot prints. */
+const REF_ATTRIBUTE = /\[ref=(e\d+)\]/;
+
+/** The `[box=x,y,width,height]` a snapshot taken with `boxes` prints. */
+const BOX_ATTRIBUTE = /\[box=(-?[\d.]+),(-?[\d.]+),(-?[\d.]+),(-?[\d.]+)\]/;
+
 /**
  * Keep the nodes a player could act on or read, and drop the structure in
  * between. An aria snapshot prints every wrapper (`- generic:`, `- list:`, a
  * nameless `- img`) and the attribute lines under links (`- /url: ...`), none
  * of which tell a persona anything, and all of which cost tokens every turn.
  *
- * Indentation is dropped too: the persona targets nodes by role and name, not
- * by depth. Text that is not an aria snapshot (the innerText fallback for a
- * thin page) is returned unchanged.
+ * Indentation is dropped too: the persona targets nodes by ref, role and
+ * name, not by depth. Text that is not an aria snapshot (the innerText
+ * fallback for a thin page) is returned unchanged.
  *
  * @param {string} text
+ * @param {{viewport?: {width: number, height: number}}} [options]
  * @returns {string}
  */
-export function compactAriaSnapshot(text) {
-  const nodes = compactAriaNodes(text);
+export function compactAriaSnapshot(text, options = {}) {
+  const nodes = compactAriaNodes(text, options);
   return nodes === null ? String(text == null ? '' : text) : nodes.map(node => node.line).join('\n');
 }
 
+/**
+ * The centre of a box, or null when the node is not visible in the viewport.
+ * Rounded, and bounded by the viewport, so a coordinate the persona copies is
+ * always one the mouse can reach.
+ *
+ * @param {Array<number>|null} box [x, y, width, height]
+ * @param {{width: number, height: number}} viewport
+ * @returns {{x: number, y: number}|null}
+ */
+export function visibleCentre(box, viewport = DEFAULT_VIEWPORT) {
+  if (!box) return null;
+  const [x, y, width, height] = box;
+  if (!(width > 0 && height > 0)) return null;
+  const cx = Math.round(x + width / 2);
+  const cy = Math.round(y + height / 2);
+  if (cx < 0 || cy < 0 || cx > viewport.width || cy > viewport.height) return null;
+  return { x: cx, y: cy };
+}
+
+/** Roles whose current value the snapshot carries, read from the DOM. */
+export const VALUE_ROLES = new Set(['textbox', 'searchbox', 'combobox', 'listbox', 'checkbox', 'radio', 'switch', 'slider', 'spinbutton', 'menuitemcheckbox', 'menuitemradio']);
+
+/** The longest value shown for a control; a longer one is cut with "...". */
+export const MAX_VALUE_CHARS = 80;
+
+/** How many controls on one screen have their value read; a typed field past it shows no value at all. */
+export const MAX_VALUE_READS = 60;
+
+/** What a password field's value is always shown as, empty or not. */
+export const MASKED_VALUE = '(hidden)';
+
+/**
+ * Roles whose snapshot text can be what the user typed. Their printed value is
+ * removed before anything is compacted, and a value is shown only when the DOM
+ * answered and said the field is not a password: Playwright's snapshot prints
+ * a password field's contents in clear, and 0.3.0 sent that to the model.
+ */
+export const TYPED_VALUE_ROLES = new Set(['textbox', 'searchbox', 'combobox', 'spinbutton']);
+
+/**
+ * A node body with the value the snapshot printed after its name removed:
+ * `textbox "Pass" [ref=e6]: hunter22` becomes `textbox "Pass" [ref=e6]`. A
+ * trailing `:` (children follow) is kept.
+ *
+ * @param {string} body
+ * @returns {string}
+ */
+export function withoutPrintedValue(body) {
+  const match = String(body).match(/^([a-z]+(?: "[^"]*")?(?: \[[^\]]*\])*)(:?)(.*)$/s);
+  if (!match) return String(body);
+  return match[3].trim() ? match[1] : `${match[1]}${match[2]}`;
+}
+
+/**
+ * A control's current state as the snapshot shows it: `value="..."` for a
+ * field or a slider, `checked` or `not checked` for a checkbox, radio or
+ * switch, `selected option="..."` for a select. A password is masked, and a
+ * value over {@link MAX_VALUE_CHARS} is cut.
+ *
+ * @param {{kind: string, value?: string, checked?: boolean|'mixed', password?: boolean, selected?: Array<string>}} state
+ * @returns {string|null}
+ */
+export function formatControlState(state) {
+  if (!state || typeof state !== 'object') return null;
+  const quote = value => {
+    let text = String(value == null ? '' : value);
+    if (text.length > MAX_VALUE_CHARS) text = `${text.slice(0, MAX_VALUE_CHARS - 3)}...`;
+    return JSON.stringify(text);
+  };
+  if (state.kind === 'check') return state.checked === 'mixed' ? 'partly checked' : state.checked ? 'checked' : 'not checked';
+  if (state.kind === 'select') return `selected option=${quote((state.selected || []).join(', '))}`;
+  if (state.kind === 'value') return state.password ? `value=${MASKED_VALUE}` : `value=${quote(state.value)}`;
+  return null;
+}
+
+/**
+ * Read a control's state in the page. Runs in the browser, serialised by
+ * Playwright, so it must not close over anything.
+ *
+ * @param {Element} el
+ * @returns {{kind: string, value?: string, checked?: boolean|'mixed', password?: boolean, selected?: Array<string>}|null}
+ */
+function readControlState(el) {
+  if (!el) return null;
+  const tag = String(el.tagName || '').toLowerCase();
+  const type = String(el.getAttribute('type') || '').toLowerCase();
+  const role = String(el.getAttribute('role') || '').toLowerCase();
+  if (tag === 'input' && (type === 'checkbox' || type === 'radio')) return { kind: 'check', checked: el.indeterminate ? 'mixed' : Boolean(el.checked) };
+  if (tag === 'select') return { kind: 'select', selected: Array.from(el.selectedOptions || []).map(option => String(option.label || option.textContent || '').trim()) };
+  if (tag === 'input' || tag === 'textarea') return { kind: 'value', value: String(el.value == null ? '' : el.value), password: tag === 'input' && type === 'password' };
+  const ariaChecked = el.getAttribute('aria-checked');
+  if (ariaChecked !== null || ['checkbox', 'radio', 'switch', 'menuitemcheckbox', 'menuitemradio'].includes(role)) {
+    return { kind: 'check', checked: ariaChecked === 'mixed' ? 'mixed' : ariaChecked === 'true' };
+  }
+  if (el.isContentEditable) return { kind: 'value', value: String(el.innerText || '').replace(/\s+/g, ' ').trim() };
+  const now = el.getAttribute('aria-valuetext') || el.getAttribute('aria-valuenow');
+  if (now !== null) return { kind: 'value', value: String(now) };
+  return null;
+}
+
+/**
+ * The refs of the controls in an AI mode snapshot whose value is worth
+ * reading, in page order and bounded by {@link MAX_VALUE_READS}.
+ *
+ * @param {string} text
+ * @returns {Array<string>}
+ */
+export function valueRefs(text) {
+  const refs = [];
+  for (const line of String(text == null ? '' : text).split('\n')) {
+    const raw = unquoteKey(line.trim().replace(/^- /, ''));
+    const role = (raw.match(/^([a-z]+)/) || [])[1] || '';
+    if (!VALUE_ROLES.has(role)) continue;
+    const ref = (raw.match(REF_ATTRIBUTE) || [])[1];
+    if (ref && !refs.includes(ref)) refs.push(ref);
+    if (refs.length >= MAX_VALUE_READS) break;
+  }
+  return refs;
+}
+
+/** How long reading one control's value may take before it is skipped. */
+export const VALUE_READ_TIMEOUT_MS = 1000;
+
+/**
+ * Read the current state of every control in a snapshot from the DOM, by
+ * ref. A control that cannot answer (gone, detached, a frame) is left out
+ * and shows what the snapshot printed.
+ *
+ * @param {any} page
+ * @param {string} text an AI mode snapshot
+ * @returns {Promise<Map<string, Object>>}
+ */
+export async function readControlValues(page, text) {
+  const values = new Map();
+  const refs = valueRefs(text);
+  if (refs.length === 0 || !page || typeof page.locator !== 'function') return values;
+  await Promise.all(refs.map(async ref => {
+    try {
+      const locator = page.locator(`aria-ref=${ref}`);
+      if (typeof locator.evaluate !== 'function') return;
+      const state = await locator.evaluate(readControlState, null, { timeout: VALUE_READ_TIMEOUT_MS });
+      if (state && typeof state === 'object') values.set(ref, state);
+    } catch {
+      // Not answerable now; the snapshot's own text stands for it.
+    }
+  }));
+  return values;
+}
+
 /**
  * The kept nodes of an aria snapshot with the depth each one had, or null for
  * text that is not an aria snapshot. The depth is not shown to the model; it
  * is what tells repeated siblings apart from a panel that follows them.
  *
+ * An interactive node from Playwright's AI mode snapshot carries its short ref
+ * (`[e12]`), which `click` and `type` accept as a target, and, when it is on
+ * screen, its centre `@(x,y)`. Two interactive nodes with the same role and
+ * name each get a hint naming the nearest landmark or named container around
+ * them ("in header", "in main"), so the persona can tell them apart.
+ *
+ * `hashLine` is the line without the centre: what "changed" and "new since
+ * the last screen" compare, so a scroll or an animation does not read as a
+ * new screen.
+ *
  * @param {string} text
- * @returns {Array<{depth: number, line: string, body: string, role: string}>|null}
+ * @param {{viewport?: {width: number, height: number}}} [options]
+ * @returns {Array<{depth: number, line: string, hashLine: string, body: string, role: string, name: string|null, ref: string|null, centre: Object|null, hint: string|null, key: string|null, duplicate: boolean}>|null}
  */
-export function compactAriaNodes(text) {
+export function compactAriaNodes(text, { viewport = DEFAULT_VIEWPORT, values = null } = {}) {
   const value = String(text == null ? '' : text);
   const lines = value.split('\n');
   if (!lines.some(line => /^\s*- /.test(line))) return null;
   const kept = [];
+  const ancestors = [];
   for (const line of lines) {
     const trimmed = line.trim();
     if (!trimmed.startsWith('- ')) continue;
-    const body = unquoteKey(trimmed.slice(2));
-    if (body.startsWith('/')) continue;
-    const role = (body.match(/^([a-z]+)/) || [])[1] || '';
+    const depth = line.length - line.trimStart().length;
+    const raw = unquoteKey(trimmed.slice(2));
+    while (ancestors.length > 0 && ancestors[ancestors.length - 1].depth >= depth) ancestors.pop();
+    if (raw.startsWith('/')) continue;
+    const role = (raw.match(/^([a-z]+)/) || [])[1] || '';
+    const name = (raw.match(/^[a-z]+ "([^"]*)"/) || [])[1] ?? null;
+    const ref = (raw.match(REF_ATTRIBUTE) || [])[1] || null;
+    const boxMatch = raw.match(BOX_ATTRIBUTE);
+    const box = boxMatch ? boxMatch.slice(1, 5).map(Number) : null;
+    // A field's printed value goes before anything else reads the line: only
+    // a value the DOM confirmed (and never a password) is shown, below.
+    const body = TYPED_VALUE_ROLES.has(role) ? withoutPrintedValue(raw.replace(NOISE_ATTRIBUTE, '')) : raw.replace(NOISE_ATTRIBUTE, '');
+    ancestors.push({ depth, role, name });
     const named = /^[a-z]+ "[^"]*"/.test(body);
     const inlineText = /^[a-z]+(?: "[^"]*")?(?: \[[^\]]*\])*: \S/.test(body);
-    if (INTERACTIVE_ROLES.has(role) || named || inlineText) {
-      kept.push({ depth: line.length - line.trimStart().length, line: `- ${body}`, body, role });
+    const interactive = INTERACTIVE_ROLES.has(role);
+    if (!(interactive || named || inlineText)) continue;
+    const container = ancestors.slice(0, -1).reverse().find(entry => LANDMARK_LABELS[entry.role] || entry.name);
+    const hint = container
+      ? (container.name ? `${container.name}${LANDMARK_LABELS[container.role] && container.role !== 'region' ? ` ${LANDMARK_LABELS[container.role]}` : ''}` : LANDMARK_LABELS[container.role])
+      : null;
+    const state = interactive && ref && values && typeof values.get === 'function' && VALUE_ROLES.has(role) ? values.get(ref) : null;
+    kept.push({
+      depth,
+      body,
+      role,
+      name,
+      state: state ? formatControlState(state) : null,
+      ref: interactive ? ref : null,
+      centre: interactive ? visibleCentre(box, viewport) : null,
+      hint,
+      key: interactive && name ? `${role}|${name}` : null,
+      duplicate: false,
+      line: '',
+      hashLine: ''
+    });
+  }
+  const counts = new Map();
+  for (const node of kept) if (node.key) counts.set(node.key, (counts.get(node.key) || 0) + 1);
+  for (const node of kept) {
+    node.duplicate = Boolean(node.key) && counts.get(node.key) > 1;
+    const split = node.body.match(/^([a-z]+(?: "[^"]*")?(?: \[[^\]]*\])*)(.*)$/s) || [null, node.body, ''];
+    let head = split[1];
+    let tail = split[2];
+    if (node.state) {
+      // The value read from the DOM replaces whatever the snapshot printed
+      // after the name and its checked attribute, so there is one answer, and
+      // it is the one the page holds.
+      head = head.replace(/ \[(?:checked|selected)(?:=[^\]]*)?\]/g, '');
+      tail = ` ${node.state}`;
     }
+    const refPart = node.ref ? ` [${node.ref}]` : '';
+    const hintPart = node.duplicate && node.hint ? ` (in ${node.hint})` : '';
+    const centrePart = node.centre ? ` @(${node.centre.x},${node.centre.y})` : '';
+    node.hashLine = `- ${head}${refPart}${hintPart}${tail}`;
+    node.line = `- ${head}${refPart}${centrePart}${hintPart}${tail}`;
   }
   return kept;
 }
@@ -151,6 +393,8 @@ function nodeShape(body) {
  * @returns {{entries: Array<{line: string, role: string, marker?: boolean}>, collapsed: number}}
  */
 export function collapseRepeatedSiblings(nodes) {
+  const keyCounts = new Map();
+  for (const node of nodes) if (node.key) keyCounts.set(node.key, (keyCounts.get(node.key) || 0) + 1);
   const unitEnd = start => {
     let end = start + 1;
     while (end < nodes.length && nodes[end].depth > nodes[start].depth) end += 1;
@@ -171,15 +415,36 @@ export function collapseRepeatedSiblings(nodes) {
       next = end;
     }
     if (units.length >= COLLAPSE_MIN_RUN) {
-      for (const [start, end] of units.slice(0, COLLAPSE_KEEP)) {
-        for (const node of nodes.slice(start, end)) entries.push({ line: node.line, role: node.role });
+      // A node whose role and name also appear outside this run is never
+      // hidden in the count: two buttons called "Run the week", one in the
+      // header and one in the main panel, both stay visible with their refs.
+      const inRun = new Map();
+      for (const node of nodes.slice(i, next)) if (node.key) inRun.set(node.key, (inRun.get(node.key) || 0) + 1);
+      // Only the first in-run twin of a name that also appears outside the
+      // run is kept, beside the first two items. One "View" button in the
+      // header must not stop a list of 249 "View" cards from collapsing.
+      const outside = key => key && (keyCounts.get(key) || 0) > (inRun.get(key) || 0);
+      const shownKeys = new Set();
+      let more = 0;
+      units.forEach((unit, index) => {
+        const unitNodes = nodes.slice(unit[0], unit[1]);
+        const twins = unitNodes.map(node => node.key).filter(key => outside(key) && !shownKeys.has(key));
+        if (index < COLLAPSE_KEEP || twins.length > 0) {
+          for (const node of unitNodes) {
+            entries.push({ line: node.line, hashLine: node.hashLine, role: node.role, key: node.key });
+            if (node.key) shownKeys.add(node.key);
+          }
+        } else {
+          more += 1;
+        }
+      });
+      if (more > 0) {
+        collapsed += more;
+        entries.push({ line: `- ... and ${more} more like it (${nodes[i].role || 'items'}; target one by its name)`, role: '', marker: true });
       }
-      const more = units.length - COLLAPSE_KEEP;
-      collapsed += more;
-      entries.push({ line: `- ... and ${more} more like it (${nodes[i].role || 'items'}; target one by its name)`, role: '', marker: true });
       i = next;
     } else {
-      entries.push({ line: nodes[i].line, role: nodes[i].role });
+      entries.push({ line: nodes[i].line, hashLine: nodes[i].hashLine, role: nodes[i].role, key: nodes[i].key });
       i += 1;
     }
   }
@@ -204,12 +469,12 @@ export function collapseRepeatedSiblings(nodes) {
  * @param {{max?: number, previous?: Set<string>|null}} [options]
  * @returns {{text: string, lines: Array<string>}|null} null when the text is not an aria snapshot
  */
-export function fitAriaSnapshot(text, { max = WEB_OBSERVATION_MAX_CHARS, previous = null } = {}) {
-  const nodes = compactAriaNodes(text);
+export function fitAriaSnapshot(text, { max = WEB_OBSERVATION_MAX_CHARS, previous = null, viewport = DEFAULT_VIEWPORT, values = null } = {}) {
+  const nodes = compactAriaNodes(text, { viewport, values });
   if (nodes === null) return null;
-  const lines = nodes.map(node => node.line);
-  const full = lines.join('\n');
-  if (full.length <= max) return { text: full, lines };
+  const lines = nodes.map(node => node.hashLine);
+  const full = nodes.map(node => node.line).join('\n');
+  if (full.length <= max) return { text: full, lines, nodes };
 
   const { entries, collapsed } = collapseRepeatedSiblings(nodes);
   const header = (left, out) => {
@@ -220,7 +485,7 @@ export function fitAriaSnapshot(text, { max = WEB_OBSERVATION_MAX_CHARS, previou
   };
   const collapsedText = entries.map(entry => entry.line).join('\n');
   const withHeader = `${header(collapsed, 0)}\n${collapsedText}`;
-  if (withHeader.length <= max) return { text: withHeader, lines };
+  if (withHeader.length <= max) return { text: withHeader, lines, nodes };
 
   const rank = entry => {
     if (entry.marker) return 0;
@@ -229,7 +494,7 @@ export function fitAriaSnapshot(text, { max = WEB_OBSERVATION_MAX_CHARS, previou
     if (INTERACTIVE_ROLES.has(entry.role) || /^- [a-z]+ "/.test(entry.line)) return 3;
     return 5;
   };
-  const isNew = entry => Boolean(previous) && !entry.marker && !previous.has(entry.line);
+  const isNew = entry => Boolean(previous) && !entry.marker && !previous.has(entry.hashLine || entry.line);
   // The last interactive nodes on the page are usually its form or its
   // confirm button (a panel after a list). When the list could not be
   // collapsed they would otherwise lose to the list on page order, so the
@@ -256,9 +521,21 @@ export function fitAriaSnapshot(text, { max = WEB_OBSERVATION_MAX_CHARS, previou
     chosen.add(candidate.index);
     used += cost;
   }
+  // Duplicates by name travel together: showing one "Run the week" and not
+  // the other is how a persona ends up clicking a name that matches twice.
+  const chosenKeys = new Set(entries.filter((entry, index) => chosen.has(index) && entry.key).map(entry => entry.key));
+  // Bounded by the cap's own slack: a page whose every card shares a name
+  // is the trim's job, not this one's.
+  entries.forEach((entry, index) => {
+    if (!entry.key || !chosenKeys.has(entry.key) || chosen.has(index)) return;
+    const cost = entry.line.length + 1;
+    if (used + cost > budget) return;
+    chosen.add(index);
+    used += cost;
+  });
   const kept = entries.filter((entry, index) => chosen.has(index)).map(entry => entry.line);
   const out = entries.length - kept.length;
-  return { text: truncateObservation(`${header(collapsed, out)}\n${kept.join('\n')}`, max), lines };
+  return { text: truncateObservation(`${header(collapsed, out)}\n${kept.join('\n')}`, max), lines, nodes };
 }
 
 /**
@@ -420,6 +697,78 @@ export function isLoopback(hostname) {
   return LOOPBACK_HOSTS.includes(hostname) || /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(hostname);
 }
 
+/**
+ * A target written as locator code rather than as something on screen. A
+ * persona that writes `getByTestId('strip-advance')` means a selector, and
+ * searching the page for that literal text waits the whole action timeout
+ * for nothing.
+ */
+export const LOCATOR_CODE = /^\s*(?:getBy|page\.|locator\(|css=|xpath=|\/\/)/;
+
+/** What the persona is told when it writes locator code as a target. */
+export const LOCATOR_CODE_MESSAGE = 'targets are refs or visible text, not locator code';
+
+export { STALE_REF_MESSAGE };
+
+/** A ref as the snapshot prints it. */
+export const REF_PATTERN = /^e\d{1,6}$/;
+
+/**
+ * A failure on the driver's side, not the game's: a target that names two
+ * elements, a target that is not a target, a ref that is not on screen. The
+ * transcript records these as `driver_error`, and the report step is told
+ * they are never evidence of a game bug.
+ */
+export class TargetError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'TargetError';
+    this.driverSide = true;
+  }
+}
+
+/**
+ * The candidates a target could mean, from the last snapshot: the
+ * interactive nodes whose name is the target (or whose role and name are,
+ * for `button "Start"`), with their refs and hints.
+ *
+ * @param {Array<Object>|null} nodes
+ * @param {string} target
+ * @returns {Array<Object>}
+ */
+export function candidatesFor(nodes, target) {
+  if (!Array.isArray(nodes)) return [];
+  const value = String(target).trim();
+  const roleAndName = value.match(/^([a-z]+)\s+"([^"]*)"$/);
+  return nodes.filter(node => node.key && node.ref && (roleAndName
+    ? node.role === roleAndName[1] && node.name === roleAndName[2]
+    : String(node.name).trim().toLowerCase() === value.toLowerCase()));
+}
+
+/** The most candidates an ambiguity answer lists before "and N more". */
+export const AMBIGUITY_LIST_MAX = 5;
+
+/**
+ * "2 elements match "Run the week"; use ref e3 (in header) or e5 (in main)".
+ *
+ * @param {string} target
+ * @param {Array<Object>} candidates
+ * @param {number} [count] how many Playwright said matched, when known
+ * @returns {string}
+ */
+export function ambiguityMessage(target, candidates, count = null) {
+  const total = Math.max(count || 0, candidates.length);
+  const shown = candidates.slice(0, AMBIGUITY_LIST_MAX);
+  const listed = shown.map(node => `${node.ref}${node.hint ? ` (in ${node.hint})` : ''}`);
+  const more = candidates.length - shown.length;
+  const choice = more > 0
+    ? `${listed.join(', ')} and ${more} more`
+    : listed.length > 1 ? `${listed.slice(0, -1).join(', ')} or ${listed[listed.length - 1]}` : listed[0];
+  return listed.length > 0
+    ? `${total} elements match ${JSON.stringify(String(target))}; use ref ${choice}`
+    : `${total} elements match ${JSON.stringify(String(target))}; take a snapshot and use the ref of the one you mean`;
+}
+
 /**
  * Turn what the model wrote into a Playwright locator.
  *
@@ -435,6 +784,7 @@ export function isLoopback(hostname) {
  */
 export function resolveLocator(page, target) {
   const value = String(target).trim();
+  if (LOCATOR_CODE.test(value)) throw new TargetError(LOCATOR_CODE_MESSAGE);
   if (KNOWN_SELECTOR_ENGINES.test(value)) return page.locator(value);
 
   // The name group cannot contain a quote (`[^"]*`), so there is nothing to
@@ -448,6 +798,16 @@ export function resolveLocator(page, target) {
   return page.getByText(value, { exact: false }).first();
 }
 
+/**
+ * Give the document a random token. Runs in the browser at the start of every
+ * new document, so a reload or a navigation gets a new one and a single page
+ * app's route change keeps it. Serialised by Playwright, so it must not close
+ * over anything.
+ */
+function markDocument() {
+  globalThis.__ravensightDoc = `${Math.random().toString(36).slice(2)}${Date.now().toString(36)}`;
+}
+
 /**
  * Hand the run's playtest identity to the page. Runs in the browser, before
  * any of the game's own scripts, so the Ravensight JS SDK finds it at
@@ -492,7 +852,18 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
   let lastObservedHash = null;
   /** The compacted lines of the last observation, so a cut can prefer what is new. */
   let lastObservedLines = null;
+  /** The kept nodes of the last observation: refs, names and hints, for a target that matches twice. */
+  let lastNodes = null;
+  let viewportSize = DEFAULT_VIEWPORT;
+  // Refs are numbered per document: after a navigation or a reload, e12 can
+  // be a different element. Each document gets a random token from an init
+  // script; a pushState or hash route change keeps it, a new document does
+  // not. The token the last observation (and so its refs) came from is
+  // compared with the current one before any ref is used.
+  let lastNodesDocument = null;
   const consoleBuffer = [];
+  /** Recent failed requests, newest last, bounded: what a console line is matched against. */
+  const networkFailures = [];
 
   /** @returns {any} */
   function requirePage() {
@@ -509,8 +880,23 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
     const current = requirePage();
     let text = '';
     let aria = true;
+    let values = null;
     const body = current.locator('body');
-    if (typeof body.ariaSnapshot === 'function') {
+    // The page level AI mode snapshot carries a ref per element, which
+    // `aria-ref=e12` resolves, and each element's box. Older Playwright
+    // builds have neither, and the body snapshot is the fallback.
+    if (typeof current.ariaSnapshot === 'function') {
+      try {
+        text = await current.ariaSnapshot({ mode: 'ai', boxes: true, timeout: ACTION_TIMEOUT_MS });
+      } catch {
+        text = '';
+      }
+      // What each field, box and select holds now, from the DOM rather than
+      // the snapshot's text: the Texas run typed a coach name twice and
+      // filed a bug because the observation kept showing the field empty.
+      if (text) values = await readControlValues(current, text);
+    }
+    if (!text && typeof body.ariaSnapshot === 'function') {
       try {
         text = await body.ariaSnapshot({ timeout: ACTION_TIMEOUT_MS });
       } catch {
@@ -532,7 +918,7 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
         // is the honest answer, and the screenshot fallback below covers it.
       }
     }
-    return { text, aria };
+    return { text, aria, values };
   }
 
   /**
@@ -545,10 +931,10 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
    */
   function shownText(snap) {
     if (snap.aria) {
-      const fitted = fitAriaSnapshot(snap.text, { max: WEB_OBSERVATION_MAX_CHARS, previous: lastObservedLines ? new Set(lastObservedLines) : null });
+      const fitted = fitAriaSnapshot(snap.text, { max: WEB_OBSERVATION_MAX_CHARS, previous: lastObservedLines ? new Set(lastObservedLines) : null, viewport: viewportSize, values: snap.values });
       if (fitted) return fitted;
     }
-    return { text: truncateObservation(snap.text, WEB_OBSERVATION_MAX_CHARS), lines: null };
+    return { text: truncateObservation(snap.text, WEB_OBSERVATION_MAX_CHARS), lines: null, nodes: null };
   }
 
   /**
@@ -580,7 +966,7 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
    */
   async function typeText(current, action) {
     const text = String(action.text == null ? '' : action.text);
-    const field = action.target ? resolveLocator(current, action.target) : null;
+    const field = action.target || action.ref ? await targetFor(current, action) : null;
     if (action.replace) {
       try {
         await (field || current.locator('*:focus')).fill(text, { timeout: field ? ACTION_TIMEOUT_MS : FOCUSED_FILL_TIMEOUT_MS });
@@ -598,8 +984,75 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
     if (text) await current.keyboard.type(text, { delay: 20 });
   }
 
+  /**
+   * The locator a click or a typing target means. A ref is preferred and
+   * checked against the page; locator code is refused at once, with no wait;
+   * a name that matches more than one interactive node on the last screen is
+   * answered with the candidates and their refs rather than guessed.
+   *
+   * @param {any} current
+   * @param {{ref?: string, target?: string}} action
+   */
+  async function targetFor(current, action) {
+    if (action.ref !== undefined && action.ref !== null && action.ref !== '') {
+      const ref = String(action.ref).trim();
+      if (!REF_PATTERN.test(ref)) throw new TargetError(`a ref looks like e12, as the snapshot prints it; ${JSON.stringify(ref)} is not one`);
+      await assertSameDocument(current);
+      const locator = current.locator(`aria-ref=${ref}`);
+      if (typeof locator.count === 'function' && (await locator.count()) === 0) {
+        throw new TargetError(`ref ${ref} is not on the current screen; take a snapshot and use a ref from it`);
+      }
+      return locator;
+    }
+    const target = String(action.target == null ? '' : action.target);
+    const candidates = candidatesFor(lastNodes, target);
+    if (candidates.length > 1) throw new TargetError(ambiguityMessage(target, candidates));
+    if (LOCATOR_CODE.test(target)) {
+      // Text that only looks like code: a node on screen really is named
+      // that, so it is clicked by its ref rather than refused.
+      if (candidates.length === 1) {
+        await assertSameDocument(current);
+        return current.locator(`aria-ref=${candidates[0].ref}`);
+      }
+      throw new TargetError(LOCATOR_CODE_MESSAGE);
+    }
+    return resolveLocator(current, target);
+  }
+
+  /**
+   * The current document's token, read from the main frame only (a child
+   * frame has its own document and its own token), or null when the page
+   * cannot answer.
+   */
+  async function documentToken(current) {
+    try {
+      const token = await current.evaluate(() => globalThis.__ravensightDoc || null);
+      return typeof token === 'string' && token ? token : null;
+    } catch {
+      return null;
+    }
+  }
+
+  /** Refuse a ref when the document changed since the observation it came from. */
+  async function assertSameDocument(current) {
+    const now = await documentToken(current);
+    if (lastNodesDocument && now !== lastNodesDocument) throw new TargetError(STALE_REF_MESSAGE);
+  }
+
+  /** The hash of the screen as it is now, on observe()'s basis. */
+  async function currentHash(current) {
+    const snap = await snapshotText();
+    const after = { text: hashText(snap, shownText(snap)) };
+    if (String(snap.text).trim().length < THIN_OBSERVATION_CHARS) {
+      const frame = await current.screenshot({ type: 'png', timeout: ACTION_TIMEOUT_MS });
+      after.screenshotBase64 = frame.toString('base64');
+    }
+    return observationHash(after);
+  }
+
   return {
     name: 'playwright_web',
+    supportsRefs: true,
     editModifier,
     keepObservations: WEB_KEEP_OBSERVATIONS,
 
@@ -613,6 +1066,7 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
     async launch({ target: url, runDir, recordVideo = false, viewport = DEFAULT_VIEWPORT }) {
       if (!url) throw new DriverError('playwright_web needs a target URL', { fatal: true });
       target = url;
+      viewportSize = viewport || DEFAULT_VIEWPORT;
       runDirectory = runDir;
       videoRequested = Boolean(recordVideo);
 
@@ -636,11 +1090,41 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
           persona: playtest.persona || null
         });
       }
+      // One token per document: set again by every new document, kept by
+      // pushState and hash route changes, which do not run init scripts.
+      if (typeof context.addInitScript === 'function') await context.addInitScript(markDocument);
       page = await context.newPage();
       page.on('console', message => {
-        if (message.type() === 'error') consoleBuffer.push({ type: 'console', text: message.text() });
+        if (message.type() !== 'error') return;
+        let location = null;
+        try {
+          location = typeof message.location === 'function' ? message.location() : null;
+        } catch {
+          location = null;
+        }
+        consoleBuffer.push({ type: 'console', text: message.text(), ...(location && location.url ? { locationUrl: String(location.url) } : {}) });
       });
       page.on('pageerror', error => consoleBuffer.push({ type: 'pageerror', text: String(error && error.message) }));
+      // The failing request behind a "Failed to load resource" line, which
+      // the console text alone never names.
+      page.on('response', response => {
+        try {
+          const status = response.status();
+          if (status < 400) return;
+          const request = typeof response.request === 'function' ? response.request() : null;
+          rememberNetworkFailure(networkFailures, { url: response.url(), status, method: request && typeof request.method === 'function' ? request.method() : null });
+        } catch {
+          // A response that cannot be read is not worth a note.
+        }
+      });
+      page.on('requestfailed', request => {
+        try {
+          const failure = typeof request.failure === 'function' ? request.failure() : null;
+          rememberNetworkFailure(networkFailures, { url: request.url(), status: null, method: typeof request.method === 'function' ? request.method() : null, error: failure && failure.errorText ? String(failure.errorText) : null });
+        } catch {
+          // Same: nothing to record.
+        }
+      });
 
       const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
       // A 404 is not something to guess around: the build is not there, and
@@ -692,6 +1176,8 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
       // because a fit keeps new nodes first), the truncated text otherwise.
       lastHash = observationHash({ text: hashText(snap, shown), screenshotBase64: observation.screenshotBase64 });
       lastObservedLines = shown.lines;
+      lastNodes = shown.nodes || null;
+      lastNodesDocument = await documentToken(current);
       // The same screen twice is sent as a marker rather than in full
       // (./tools.js), which is most of what a stuck persona would otherwise
       // pay for.
@@ -717,11 +1203,11 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
             if (action.submit) await current.keyboard.press('Enter');
             break;
           case 'click': {
-            if (action.target && typeof action.target === 'object') {
+            if (!action.ref && action.target && typeof action.target === 'object') {
               await current.mouse.click(Number(action.target.x), Number(action.target.y));
               break;
             }
-            await resolveLocator(current, action.target).click({ timeout: ACTION_TIMEOUT_MS });
+            await (await targetFor(current, action)).click({ timeout: ACTION_TIMEOUT_MS });
             break;
           }
           case 'navigate': {
@@ -739,20 +1225,44 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
             return { ok: false, changed: false, error: `the playwright_web driver cannot do "${action.kind}"` };
         }
       } catch (error) {
-        return { ok: false, changed: false, error: String(error && error.message ? error.message : error) };
+        // Playwright colours its call log; the colour codes are noise here.
+        const text = String(error && error.message ? error.message : error).replace(/\u001b\[[0-9;]*m/g, '');
+        // Two elements under one name: Playwright's strict mode. Answered
+        // with the candidates and their refs, so the next try can work.
+        if (/strict mode violation/i.test(text)) {
+          const count = Number((text.match(/resolved to (\d+) elements/) || [])[1]) || null;
+          const candidates = action.target && typeof action.target === 'string' ? candidatesFor(lastNodes, action.target) : [];
+          return { ok: false, changed: false, error: ambiguityMessage(action.target || action.ref || '', candidates, count), driverError: true };
+        }
+        if (error && error.driverSide) return { ok: false, changed: false, error: text, driverError: true };
+        // A click the page kept intercepting is the game's: something sat on
+        // top of the target and never went away. Named, never a driver error.
+        const interceptor = text.match(/^\s*-?\s*(.+?)\s+(?:from .+? subtree\s+)?intercepts pointer events/m);
+        const intercepted = interceptor ? interceptor[1] : null;
+        if (/Timeout \d+ms exceeded/i.test(text) && intercepted) {
+          return { ok: false, changed: false, error: `timed out: ${intercepted.slice(0, 160)} intercepts pointer events over the target`, driverError: false };
+        }
+        // A timeout with the page unchanged while resolving a click or typing
+        // target means the target was never found or never actionable: the
+        // driver's failure, not the game's. Never for navigate: a game that
+        // never loads is a real blocker.
+        if (/Timeout \d+ms exceeded/i.test(text) && (action.kind === 'click' || action.kind === 'type')) {
+          let unchanged = false;
+          try {
+            unchanged = (await currentHash(current)) === before;
+          } catch {
+            unchanged = false;
+          }
+          return { ok: false, changed: false, error: text, driverError: unchanged };
+        }
+        return { ok: false, changed: false, error: text };
       }
 
       let changed = false;
       try {
         // The same basis observe() hashes, so the two are comparable.
         // Without it a long screen is always "changed".
-        const snap = await snapshotText();
-        const after = { text: hashText(snap, shownText(snap)) };
-        if (String(snap.text).trim().length < THIN_OBSERVATION_CHARS) {
-          const frame = await current.screenshot({ type: 'png', timeout: ACTION_TIMEOUT_MS });
-          after.screenshotBase64 = frame.toString('base64');
-        }
-        changed = observationHash(after) !== before;
+        changed = (await currentHash(current)) !== before;
       } catch {
         changed = true;
       }
@@ -775,7 +1285,7 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
 
     /** @returns {Promise<Array<{type: string, text: string}>>} */
     async consoleErrors() {
-      return consoleBuffer.splice(0, consoleBuffer.length);
+      return attachNetworkFailures(consoleBuffer.splice(0, consoleBuffer.length), networkFailures);
     },
 
     /** @returns {Promise<{videoPath: string|null}>} */
@@ -808,6 +1318,72 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
   };
 }
 
+/** How many recent failed requests are kept to match console lines against. */
+export const MAX_NETWORK_FAILURES = 50;
+
+/** The longest request URL a note carries. */
+export const MAX_NETWORK_URL_CHARS = 200;
+
+/**
+ * A request URL as a note carries it: no query string and no fragment (they
+ * carry tokens and session ids as often as anything useful), and bounded.
+ *
+ * @param {string} url
+ * @returns {string}
+ */
+export function networkUrl(url) {
+  let value = String(url == null ? '' : url);
+  try {
+    const parsed = new URL(value);
+    value = `${parsed.origin === 'null' ? `${parsed.protocol}` : parsed.origin}${parsed.pathname}`;
+  } catch {
+    value = value.split(/[?#]/)[0];
+  }
+  return value.length > MAX_NETWORK_URL_CHARS ? `${value.slice(0, MAX_NETWORK_URL_CHARS - 3)}...` : value;
+}
+
+/**
+ * @param {Array<Object>} list
+ * @param {{url: string, status: number|null, method?: string|null, error?: string|null}} failure
+ */
+export function rememberNetworkFailure(list, failure) {
+  list.push({ url: networkUrl(failure.url), status: Number.isInteger(failure.status) ? failure.status : null, method: failure.method || null, error: failure.error || null, claimed: false });
+  while (list.length > MAX_NETWORK_FAILURES) list.shift();
+}
+
+/** The console line a browser prints for a failed request. */
+const FAILED_RESOURCE = /^Failed to load resource\b/i;
+
+/**
+ * Give each "Failed to load resource" console line the request it was about:
+ * the one at the line's own location when the browser gave one, otherwise the
+ * oldest unclaimed failure with the same status (or the same network error).
+ * The line's text gains the method, the URL and the status, so the note says
+ * what failed; `url` and `status` ride along for the report.
+ *
+ * @param {Array<{type: string, text: string, locationUrl?: string}>} entries
+ * @param {Array<Object>} failures the driver's recent failed requests
+ * @returns {Array<Object>}
+ */
+export function attachNetworkFailures(entries, failures) {
+  return entries.map(entry => {
+    const { locationUrl, ...rest } = entry;
+    if (!FAILED_RESOURCE.test(String(entry.text || ''))) return rest;
+    const status = Number((String(entry.text).match(/status of (\d{3})/) || [])[1]) || null;
+    const location = locationUrl ? networkUrl(locationUrl) : null;
+    let match = location ? failures.find(failure => !failure.claimed && failure.url === location) : null;
+    if (!match) {
+      match = failures.find(failure => !failure.claimed && (status ? failure.status === status : failure.status === null && failure.error && String(entry.text).includes(failure.error)));
+    }
+    const url = match ? match.url : location;
+    if (!url) return rest;
+    if (match) match.claimed = true;
+    const shownStatus = match && match.status ? match.status : status;
+    const method = match && match.method ? `${match.method} ` : '';
+    return { ...rest, text: `${rest.text} (${method}${url}${shownStatus ? ` answered ${shownStatus}` : ''})`, url, status: shownStatus || null };
+  });
+}
+
 /**
  * Playwright names its recording with a random id. The upload allowlist and
  * the server's 90 day lifecycle rule are written against `session.webm`, so