ravensight-playtest 0.3.0 → 0.3.2

package/src/run/drivers/web.js

@@ -133,6 +133,190 @@ export function visibleCentre(box, viewport = DEFAULT_VIEWPORT) {
   return { x: cx, y: cy };
 }
 
+/** Roles whose current value the snapshot carries, read from the DOM. */
+export const VALUE_ROLES = new Set(['textbox', 'searchbox', 'combobox', 'listbox', 'checkbox', 'radio', 'switch', 'slider', 'spinbutton', 'menuitemcheckbox', 'menuitemradio']);
+
+/** The longest value shown for a control; a longer one is cut with "...". */
+export const MAX_VALUE_CHARS = 80;
+
+/** How many controls on one screen have their value read; a typed field past it shows no value at all. */
+export const MAX_VALUE_READS = 60;
+
+/** What a password field's value is always shown as, empty or not. */
+export const MASKED_VALUE = '(hidden)';
+
+/**
+ * Roles whose snapshot text can be what the user typed. Their printed value is
+ * removed before anything is compacted, and a value is shown only when the DOM
+ * answered and said the field is not a password: Playwright's snapshot prints
+ * a password field's contents in clear, and 0.3.0 sent that to the model.
+ */
+export const TYPED_VALUE_ROLES = new Set(['textbox', 'searchbox', 'combobox', 'spinbutton']);
+
+/**
+ * A node body with the value the snapshot printed after its name removed:
+ * `textbox "Pass" [ref=e6]: hunter22` becomes `textbox "Pass" [ref=e6]`. A
+ * trailing `:` (children follow) is kept.
+ *
+ * @param {string} body
+ * @returns {string}
+ */
+export function withoutPrintedValue(body) {
+  const match = String(body).match(/^([a-z]+(?: "[^"]*")?(?: \[[^\]]*\])*)(:?)(.*)$/s);
+  if (!match) return String(body);
+  return match[3].trim() ? match[1] : `${match[1]}${match[2]}`;
+}
+
+/**
+ * A control's current state as the snapshot shows it: `value="..."` for a
+ * field or a slider, `checked` or `not checked` for a checkbox, radio or
+ * switch, `selected option="..."` for a select. A password is masked, and a
+ * value over {@link MAX_VALUE_CHARS} is cut.
+ *
+ * @param {{kind: string, value?: string, checked?: boolean|'mixed', password?: boolean, selected?: Array<string>}} state
+ * @returns {string|null}
+ */
+export function formatControlState(state) {
+  if (!state || typeof state !== 'object') return null;
+  const quote = value => {
+    let text = String(value == null ? '' : value);
+    if (text.length > MAX_VALUE_CHARS) text = `${text.slice(0, MAX_VALUE_CHARS - 3)}...`;
+    return JSON.stringify(text);
+  };
+  if (state.kind === 'check') return state.checked === 'mixed' ? 'partly checked' : state.checked ? 'checked' : 'not checked';
+  if (state.kind === 'select') return `selected option=${quote((state.selected || []).join(', '))}`;
+  if (state.kind === 'value') return state.password ? `value=${MASKED_VALUE}` : `value=${quote(state.value)}`;
+  return null;
+}
+
+/**
+ * Read a control's state in the page. Runs in the browser, serialised by
+ * Playwright, so it must not close over anything.
+ *
+ * @param {Element} el
+ * @returns {{kind: string, value?: string, checked?: boolean|'mixed', password?: boolean, selected?: Array<string>}|null}
+ */
+function readControlState(el) {
+  if (!el) return null;
+  const tag = String(el.tagName || '').toLowerCase();
+  const type = String(el.getAttribute('type') || '').toLowerCase();
+  const role = String(el.getAttribute('role') || '').toLowerCase();
+  if (tag === 'input' && (type === 'checkbox' || type === 'radio')) return { kind: 'check', checked: el.indeterminate ? 'mixed' : Boolean(el.checked) };
+  if (tag === 'select') return { kind: 'select', selected: Array.from(el.selectedOptions || []).map(option => String(option.label || option.textContent || '').trim()) };
+  if (tag === 'input' || tag === 'textarea') return { kind: 'value', value: String(el.value == null ? '' : el.value), password: tag === 'input' && type === 'password' };
+  const ariaChecked = el.getAttribute('aria-checked');
+  if (ariaChecked !== null || ['checkbox', 'radio', 'switch', 'menuitemcheckbox', 'menuitemradio'].includes(role)) {
+    return { kind: 'check', checked: ariaChecked === 'mixed' ? 'mixed' : ariaChecked === 'true' };
+  }
+  if (el.isContentEditable) return { kind: 'value', value: String(el.innerText || '').replace(/\s+/g, ' ').trim() };
+  const now = el.getAttribute('aria-valuetext') || el.getAttribute('aria-valuenow');
+  if (now !== null) return { kind: 'value', value: String(now) };
+  return null;
+}
+
+/**
+ * The refs of the controls in an AI mode snapshot whose value is worth
+ * reading, in page order and bounded by {@link MAX_VALUE_READS}.
+ *
+ * @param {string} text
+ * @returns {Array<string>}
+ */
+export function valueRefs(text) {
+  const refs = [];
+  for (const line of String(text == null ? '' : text).split('\n')) {
+    const raw = unquoteKey(line.trim().replace(/^- /, ''));
+    const role = (raw.match(/^([a-z]+)/) || [])[1] || '';
+    if (!VALUE_ROLES.has(role)) continue;
+    const ref = (raw.match(REF_ATTRIBUTE) || [])[1];
+    if (ref && !refs.includes(ref)) refs.push(ref);
+    if (refs.length >= MAX_VALUE_READS) break;
+  }
+  return refs;
+}
+
+/** How long reading one control's value may take before it is skipped. */
+export const VALUE_READ_TIMEOUT_MS = 1000;
+
+/**
+ * Read the current state of every control in a snapshot from the DOM, by
+ * ref. A control that cannot answer (gone, detached, a frame) is left out
+ * and shows what the snapshot printed.
+ *
+ * @param {any} page
+ * @param {string} text an AI mode snapshot
+ * @returns {Promise<Map<string, Object>>}
+ */
+export async function readControlValues(page, text) {
+  const values = new Map();
+  const refs = valueRefs(text);
+  if (refs.length === 0 || !page || typeof page.locator !== 'function') return values;
+  await Promise.all(refs.map(async ref => {
+    try {
+      const locator = page.locator(`aria-ref=${ref}`);
+      if (typeof locator.evaluate !== 'function') return;
+      const state = await locator.evaluate(readControlState, null, { timeout: VALUE_READ_TIMEOUT_MS });
+      if (state && typeof state === 'object') values.set(ref, state);
+    } catch {
+      // Not answerable now; the snapshot's own text stands for it.
+    }
+  }));
+  return values;
+}
+
+/**
+ * What a field holds, read in the page. Runs in the browser, serialised by
+ * Playwright, so it must not close over anything.
+ *
+ * @param {Element} el
+ * @returns {string|null}
+ */
+function readFieldValue(el) {
+  if (!el) return null;
+  if ('value' in el && typeof el.value === 'string') return el.value;
+  if (el.isContentEditable) return String(el.innerText || '');
+  return null;
+}
+
+/**
+ * The value a typing target holds, for `changed` on a `type` action. It is
+ * compared and never shown, so a password field's value is read like any
+ * other and goes no further than this comparison. Null when the field cannot
+ * answer (gone, nothing focused, an older Playwright).
+ *
+ * @param {any} locator
+ * @returns {Promise<string|null>}
+ */
+export async function fieldValue(locator) {
+  if (!locator || typeof locator.evaluate !== 'function') return null;
+  try {
+    const value = await locator.evaluate(readFieldValue, null, { timeout: VALUE_READ_TIMEOUT_MS });
+    return typeof value === 'string' ? value : null;
+  } catch {
+    return null;
+  }
+}
+
+/** How long `act` waits once for a late change (a navigation still starting) before it says the screen did not change. */
+export const SETTLE_MS = 300;
+
+/**
+ * Whether an action changed anything: the URL, the document (a navigation or
+ * a reload), the compacted tree (which carries every field's value), or for
+ * `type`, the field's own value before and after. Any one is a change. The
+ * Texas round logged a click that navigated, and typing into fields, as
+ * `changed: false`, and the action log feeds the downgrade rule.
+ *
+ * @param {{urlBefore?: string|null, urlAfter?: string|null, tokenBefore?: string|null, tokenAfter?: string|null, hashBefore?: string|null, hashAfter?: string|null, valueBefore?: string|null, valueAfter?: string|null}} facts
+ * @returns {boolean}
+ */
+export function computeChanged(facts) {
+  const differs = (a, b) => a !== null && a !== undefined && b !== null && b !== undefined && a !== b;
+  return differs(facts.urlBefore, facts.urlAfter)
+    || differs(facts.tokenBefore, facts.tokenAfter)
+    || (facts.hashAfter !== undefined && facts.hashAfter !== facts.hashBefore)
+    || differs(facts.valueBefore, facts.valueAfter);
+}
+
 /**
  * The kept nodes of an aria snapshot with the depth each one had, or null for
  * text that is not an aria snapshot. The depth is not shown to the model; it
@@ -152,7 +336,7 @@ export function visibleCentre(box, viewport = DEFAULT_VIEWPORT) {
  * @param {{viewport?: {width: number, height: number}}} [options]
  * @returns {Array<{depth: number, line: string, hashLine: string, body: string, role: string, name: string|null, ref: string|null, centre: Object|null, hint: string|null, key: string|null, duplicate: boolean}>|null}
  */
-export function compactAriaNodes(text, { viewport = DEFAULT_VIEWPORT } = {}) {
+export function compactAriaNodes(text, { viewport = DEFAULT_VIEWPORT, values = null } = {}) {
   const value = String(text == null ? '' : text);
   const lines = value.split('\n');
   if (!lines.some(line => /^\s*- /.test(line))) return null;
@@ -170,7 +354,9 @@ export function compactAriaNodes(text, { viewport = DEFAULT_VIEWPORT } = {}) {
     const ref = (raw.match(REF_ATTRIBUTE) || [])[1] || null;
     const boxMatch = raw.match(BOX_ATTRIBUTE);
     const box = boxMatch ? boxMatch.slice(1, 5).map(Number) : null;
-    const body = raw.replace(NOISE_ATTRIBUTE, '');
+    // A field's printed value goes before anything else reads the line: only
+    // a value the DOM confirmed (and never a password) is shown, below.
+    const body = TYPED_VALUE_ROLES.has(role) ? withoutPrintedValue(raw.replace(NOISE_ATTRIBUTE, '')) : raw.replace(NOISE_ATTRIBUTE, '');
     ancestors.push({ depth, role, name });
     const named = /^[a-z]+ "[^"]*"/.test(body);
     const inlineText = /^[a-z]+(?: "[^"]*")?(?: \[[^\]]*\])*: \S/.test(body);
@@ -180,11 +366,13 @@ export function compactAriaNodes(text, { viewport = DEFAULT_VIEWPORT } = {}) {
     const hint = container
       ? (container.name ? `${container.name}${LANDMARK_LABELS[container.role] && container.role !== 'region' ? ` ${LANDMARK_LABELS[container.role]}` : ''}` : LANDMARK_LABELS[container.role])
       : null;
+    const state = interactive && ref && values && typeof values.get === 'function' && VALUE_ROLES.has(role) ? values.get(ref) : null;
     kept.push({
       depth,
       body,
       role,
       name,
+      state: state ? formatControlState(state) : null,
       ref: interactive ? ref : null,
       centre: interactive ? visibleCentre(box, viewport) : null,
       hint,
@@ -199,8 +387,15 @@ export function compactAriaNodes(text, { viewport = DEFAULT_VIEWPORT } = {}) {
   for (const node of kept) {
     node.duplicate = Boolean(node.key) && counts.get(node.key) > 1;
     const split = node.body.match(/^([a-z]+(?: "[^"]*")?(?: \[[^\]]*\])*)(.*)$/s) || [null, node.body, ''];
-    const head = split[1];
-    const tail = split[2];
+    let head = split[1];
+    let tail = split[2];
+    if (node.state) {
+      // The value read from the DOM replaces whatever the snapshot printed
+      // after the name and its checked attribute, so there is one answer, and
+      // it is the one the page holds.
+      head = head.replace(/ \[(?:checked|selected)(?:=[^\]]*)?\]/g, '');
+      tail = ` ${node.state}`;
+    }
     const refPart = node.ref ? ` [${node.ref}]` : '';
     const hintPart = node.duplicate && node.hint ? ` (in ${node.hint})` : '';
     const centrePart = node.centre ? ` @(${node.centre.x},${node.centre.y})` : '';
@@ -225,6 +420,13 @@ export const COLLAPSE_MIN_RUN = 4;
 /** How many of a collapsed run are still shown, so the model sees what the items look like. */
 export const COLLAPSE_KEEP = 2;
 
+/**
+ * A run of this many look-alike siblings or more is collapsed even when the
+ * page is under the cap: twenty school cards cost every turn they stay in
+ * context and tell the persona nothing the first two and a count do not.
+ */
+export const COLLAPSE_ALWAYS_RUN = 20;
+
 /**
  * The shape of one node, without the words that make it unique: its role,
  * its state attributes, whether it has a name, whether the name carries
@@ -249,9 +451,15 @@ function nodeShape(body) {
  * says how many more there are.
  *
  * @param {Array<{depth: number, line: string, body: string, role: string}>} nodes
- * @returns {{entries: Array<{line: string, role: string, marker?: boolean}>, collapsed: number}}
+ * @param {{minRun?: number, keepNamedTwins?: boolean}} [options] the shortest
+ *   run that collapses; with `keepNamedTwins`, a run in which two units share
+ *   a control's role and name is never collapsed (the 0.3.1 rule that
+ *   same-named controls are shown together, each with its ref)
+ * @returns {{entries: Array<{line: string, role: string, marker?: boolean}>, collapsed: number, longest: number}}
+ *   `longest` is the most look-alike siblings in one run, collapsed or not, and
+ *   `longestTwins` whether two units of that run share a control's role and name
  */
-export function collapseRepeatedSiblings(nodes) {
+export function collapseRepeatedSiblings(nodes, { minRun = COLLAPSE_MIN_RUN, keepNamedTwins = false } = {}) {
   const keyCounts = new Map();
   for (const node of nodes) if (node.key) keyCounts.set(node.key, (keyCounts.get(node.key) || 0) + 1);
   const unitEnd = start => {
@@ -262,6 +470,8 @@ export function collapseRepeatedSiblings(nodes) {
   const signature = (start, end) => nodes.slice(start, end).map(node => `${node.depth - nodes[start].depth}:${nodeShape(node.body)}`).join(';');
   const entries = [];
   let collapsed = 0;
+  let longest = nodes.length > 0 ? 1 : 0;
+  let longestTwins = false;
   let i = 0;
   while (i < nodes.length) {
     const units = [[i, unitEnd(i)]];
@@ -273,7 +483,23 @@ export function collapseRepeatedSiblings(nodes) {
       units.push([next, end]);
       next = end;
     }
-    if (units.length >= COLLAPSE_MIN_RUN) {
+    let runHasTwins = false;
+    if (units.length > 1) {
+      const seen = new Map();
+      for (const node of nodes.slice(i, next)) if (node.key) seen.set(node.key, (seen.get(node.key) || 0) + 1);
+      runHasTwins = [...seen.values()].some(count => count > 1);
+    }
+    if (units.length > longest) {
+      longest = units.length;
+      longestTwins = runHasTwins;
+    }
+    const twinsInRun = keepNamedTwins && units.length >= minRun && runHasTwins;
+    if (twinsInRun) {
+      // Under the cap there is room for all of them, and collapsing would
+      // hide controls that share a name with one still shown.
+      for (const node of nodes.slice(i, next)) entries.push({ line: node.line, hashLine: node.hashLine, role: node.role, key: node.key });
+      i = next;
+    } else if (units.length >= minRun) {
       // A node whose role and name also appear outside this run is never
       // hidden in the count: two buttons called "Run the week", one in the
       // header and one in the main panel, both stay visible with their refs.
@@ -307,14 +533,17 @@ export function collapseRepeatedSiblings(nodes) {
       i += 1;
     }
   }
-  return { entries, collapsed };
+  return { entries, collapsed, longest, longestTwins };
 }
 
 /**
  * Fit a compacted aria snapshot under the web ceiling without losing the
  * part of the page the persona has to act on.
  *
- * Under the cap nothing changes. Over it, in this order: interactive nodes
+ * Under the cap only runs of {@link COLLAPSE_ALWAYS_RUN} or more look-alike
+ * siblings collapse, and never a run in which two units share a control's
+ * name (22 "View" buttons are all shown, each with its ref); everything else
+ * is shown as it is. Over it, in this order: interactive nodes
  * are what is kept (the last tenth of them on the page first, since that is
  * where a form after a long list sits, then inputs, then buttons and links,
  * then any role with a name); runs of look-alike siblings collapse to their
@@ -328,20 +557,24 @@ export function collapseRepeatedSiblings(nodes) {
  * @param {{max?: number, previous?: Set<string>|null}} [options]
  * @returns {{text: string, lines: Array<string>}|null} null when the text is not an aria snapshot
  */
-export function fitAriaSnapshot(text, { max = WEB_OBSERVATION_MAX_CHARS, previous = null, viewport = DEFAULT_VIEWPORT } = {}) {
-  const nodes = compactAriaNodes(text, { viewport });
+export function fitAriaSnapshot(text, { max = WEB_OBSERVATION_MAX_CHARS, previous = null, viewport = DEFAULT_VIEWPORT, values = null } = {}) {
+  const nodes = compactAriaNodes(text, { viewport, values });
   if (nodes === null) return null;
   const lines = nodes.map(node => node.hashLine);
   const full = nodes.map(node => node.line).join('\n');
-  if (full.length <= max) return { text: full, lines, nodes };
-
-  const { entries, collapsed } = collapseRepeatedSiblings(nodes);
   const header = (left, out) => {
     const parts = [];
     if (left > 0) parts.push(`${left} similar items collapsed`);
     if (out > 0) parts.push(`${out} lines left out to fit`);
     return parts.length > 0 ? `[${parts.join('; ')}]` : '';
   };
+  if (full.length <= max) {
+    const long = collapseRepeatedSiblings(nodes, { minRun: COLLAPSE_ALWAYS_RUN, keepNamedTwins: true });
+    if (long.collapsed === 0) return { text: full, lines, nodes };
+    return { text: `${header(long.collapsed, 0)}\n${long.entries.map(entry => entry.line).join('\n')}`, lines, nodes };
+  }
+
+  const { entries, collapsed } = collapseRepeatedSiblings(nodes);
   const collapsedText = entries.map(entry => entry.line).join('\n');
   const withHeader = `${header(collapsed, 0)}\n${collapsedText}`;
   if (withHeader.length <= max) return { text: withHeader, lines, nodes };
@@ -397,6 +630,113 @@ export function fitAriaSnapshot(text, { max = WEB_OBSERVATION_MAX_CHARS, previou
   return { text: truncateObservation(`${header(collapsed, out)}\n${kept.join('\n')}`, max), lines, nodes };
 }
 
+/**
+ * Read the screen the way a persona's observation does: the page level AI
+ * mode snapshot with each control's value from the DOM, the body snapshot on
+ * an older Playwright, and the innerText fallback for a thin page. `check`
+ * measures a first screen with this too, so what it reports is what a
+ * persona would be shown.
+ *
+ * @param {any} current a Playwright page
+ * @param {{innerTextTimeoutMs?: number}} [options] how long the innerText fallback may take
+ * @returns {Promise<{text: string, aria: boolean, values: Map<string, Object>|null}>}
+ *   the screen text, and whether it is the aria snapshot (compacted) or the
+ *   innerText fallback (never compacted: a bullet list in the page is not
+ *   snapshot structure)
+ */
+export async function readSnapshot(current, { innerTextTimeoutMs = INNER_TEXT_TIMEOUT_MS } = {}) {
+  return readSnapshotWith(current, innerTextTimeoutMs);
+}
+
+/** How long the innerText fallback may take: a page whose script is stuck cannot hang a turn. */
+export const INNER_TEXT_TIMEOUT_MS = 5000;
+
+/**
+ * A promise that rejects after `ms` if the given one has not settled.
+ *
+ * @param {Promise<any>} promise
+ * @param {number} ms
+ * @returns {Promise<any>}
+ */
+export function withTimeout(promise, ms) {
+  let timer;
+  const limit = new Promise((resolve, reject) => {
+    timer = setTimeout(() => reject(new Error(`Timeout ${ms}ms exceeded`)), ms);
+  });
+  return Promise.race([promise, limit]).finally(() => clearTimeout(timer));
+}
+
+async function readSnapshotWith(current, innerTextTimeoutMs) {
+  let text = '';
+  let aria = true;
+  let values = null;
+  const body = current.locator('body');
+  // The page level AI mode snapshot carries a ref per element, which
+  // `aria-ref=e12` resolves, and each element's box. Older Playwright
+  // builds have neither, and the body snapshot is the fallback.
+  if (typeof current.ariaSnapshot === 'function') {
+    try {
+      text = await current.ariaSnapshot({ mode: 'ai', boxes: true, timeout: ACTION_TIMEOUT_MS });
+    } catch {
+      text = '';
+    }
+    // What each field, box and select holds now, from the DOM rather than
+    // the snapshot's text: the Texas run typed a coach name twice and
+    // filed a bug because the observation kept showing the field empty.
+    if (text) values = await readControlValues(current, text);
+  }
+  if (!text && typeof body.ariaSnapshot === 'function') {
+    try {
+      text = await body.ariaSnapshot({ timeout: ACTION_TIMEOUT_MS });
+    } catch {
+      text = '';
+    }
+  }
+  if (text.trim().length < THIN_OBSERVATION_CHARS) {
+    try {
+      const inner = await withTimeout(current.evaluate(() => {
+        const el = globalThis.document && globalThis.document.body;
+        return el ? el.innerText : '';
+      }), innerTextTimeoutMs);
+      if (String(inner).trim().length > text.trim().length) {
+        text = String(inner);
+        aria = false;
+      }
+    } catch {
+      // A page mid-navigation has no body to read. The thin observation
+      // is the honest answer, and the driver's screenshot fallback covers it.
+    }
+  }
+  return { text, aria, values };
+}
+
+/**
+ * How heavy a screen is for a persona: the whole compacted text (before any
+ * fit), what the persona is actually shown after the fit, and the longest run
+ * of look-alike siblings on it.
+ *
+ * @param {{text: string, aria: boolean, values?: Map<string, Object>|null}} snap from {@link readSnapshot}
+ * @param {{max?: number, viewport?: {width: number, height: number}}} [options]
+ * @returns {{aria: boolean, fullChars: number, shownChars: number, longestRun: number, longestRunCollapsed: boolean}}
+ *   `longestRunCollapsed` says whether the persona sees that run as two and a
+ *   count, decided the way the driver decides it
+ */
+export function measureSnapshot(snap, { max = WEB_OBSERVATION_MAX_CHARS, viewport = DEFAULT_VIEWPORT } = {}) {
+  const text = String((snap && snap.text) || '');
+  const fitted = snap && snap.aria ? fitAriaSnapshot(text, { max, viewport, values: snap.values || null }) : null;
+  if (!fitted) {
+    return { aria: false, fullChars: text.length, shownChars: truncateObservation(text, max).length, longestRun: 0, longestRunCollapsed: false };
+  }
+  const full = fitted.nodes.map(node => node.line).join('\n');
+  const runs = collapseRepeatedSiblings(fitted.nodes);
+  // The driver's own decision: over the cap a run of four or more collapses;
+  // under it, only a run of COLLAPSE_ALWAYS_RUN or more with no two items
+  // sharing a control's name. The rest is shown in full.
+  const overCap = full.length > max;
+  const longestRunCollapsed = overCap ? runs.longest >= COLLAPSE_MIN_RUN : runs.longest >= COLLAPSE_ALWAYS_RUN && !runs.longestTwins;
+  return { aria: true, fullChars: full.length, shownChars: fitted.text.length, longestRun: runs.longest, longestRunCollapsed };
+}
+
 /**
  * Playwright writes a node key as a single quoted YAML scalar when the name
  * contains `: `, ` #`, a brace or a backtick: `- 'button "Player #1"'`. Strip
@@ -628,6 +968,73 @@ export function ambiguityMessage(target, candidates, count = null) {
     : `${total} elements match ${JSON.stringify(String(target))}; take a snapshot and use the ref of the one you mean`;
 }
 
+/**
+ * What a target copied from the snapshot means. A persona pastes a whole line
+ * as often as a ref: the Texas round sent `button "Run the week" @(1003,304)`
+ * as a target, and it was searched for as text for 15 seconds, twice. Any
+ * subset of `role "name" [eN] @(x,y)` (with or without the leading `- `, a
+ * `(in header)` hint or a `: value` tail) is read into its parts.
+ *
+ * The `[eN]` and `@(x,y)` are read only after the `role "name"` part, so a
+ * name that itself contains one (`link "Footnote [e3]"`, `button "Teleport
+ * @(3, 4)"`) is a name, not a ref or a point. A bare `e4` is marked `bare`:
+ * the caller treats it as text when a node on screen is named exactly that (a
+ * chess square), and as a ref otherwise.
+ *
+ * Returns null for a target that carries neither a ref nor coordinates, which
+ * is the ordinary `button "Start"` or visible text case.
+ *
+ * @param {string} target
+ * @returns {{ref: string|null, x: number|null, y: number|null, role: string|null, name: string|null, bare?: boolean}|null}
+ */
+export function parseSnapshotTarget(target) {
+  const value = String(target == null ? '' : target).trim().replace(/^- /, '');
+  if (REF_PATTERN.test(value)) return { ref: value, x: null, y: null, role: null, name: null, bare: true };
+  const roleAndName = value.match(/^([a-z]+)\s+"([^"]*)"/);
+  const bareRole = roleAndName ? null : value.match(/^([a-z]+)(?=\s*(?:\[|@|$))/);
+  const rest = roleAndName ? value.slice(roleAndName[0].length) : value;
+  const ref = (rest.match(/\[(e\d{1,6})\]/) || [])[1] || null;
+  const at = rest.match(/@\(\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\)/);
+  if (!ref && !at) return null;
+  return {
+    ref,
+    x: at ? Math.round(Number(at[1])) : null,
+    y: at ? Math.round(Number(at[2])) : null,
+    role: roleAndName ? roleAndName[1] : bareRole ? bareRole[1] : null,
+    name: roleAndName ? roleAndName[2] : null
+  };
+}
+
+/**
+ * Whether a target is text the last snapshot shows: a node's name or its
+ * inline text, ignoring case. Used to refuse at once a target that is not a
+ * ref, coordinates or anything on screen, rather than search for it for the
+ * whole action timeout.
+ *
+ * @param {Array<Object>|null} nodes
+ * @param {string} target
+ * @returns {boolean}
+ */
+export function textOnScreen(nodes, target) {
+  const value = String(target == null ? '' : target).trim().toLowerCase();
+  if (!value) return false;
+  return nodes.some(node => String(node.name || '').toLowerCase().includes(value) || String(node.body || '').toLowerCase().includes(value));
+}
+
+/** What a target that is none of a ref, coordinates or visible text is told. */
+export const NOT_A_TARGET_MESSAGE = 'is not a ref, coordinates or text on the current screen; use the ref the snapshot shows, such as e12';
+
+/**
+ * A locator narrowed to visible elements when this Playwright can: a mobile
+ * only duplicate hidden by CSS must never be the one clicked.
+ *
+ * @param {any} locator
+ * @returns {any}
+ */
+export function visibleOnly(locator) {
+  return locator && typeof locator.filter === 'function' ? locator.filter({ visible: true }) : locator;
+}
+
 /**
  * Turn what the model wrote into a Playwright locator.
  *
@@ -650,11 +1057,19 @@ export function resolveLocator(page, target) {
   // escape when it is spliced back into the `role=` engine. A target whose
   // name really does contain a quote falls through to the text search below.
   const roleAndName = value.match(/^([a-z]+)\s+"([^"]*)"$/);
-  if (roleAndName) return page.locator(`role=${roleAndName[1]}[name="${roleAndName[2]}"]`);
+  // No first(): two visible matches are Playwright's strict mode failure,
+  // which is answered with the candidates rather than guessed.
+  if (roleAndName) return visibleOnly(page.locator(`role=${roleAndName[1]}[name="${roleAndName[2]}"]`));
 
   if (/^[#.[]/.test(value) || /^[a-z][a-z0-9-]*(\[|\.|#|:| >)/i.test(value)) return page.locator(value);
 
-  return page.getByText(value, { exact: false }).first();
+  // Hidden matches are skipped: a page with a mobile only copy of a button
+  // hidden by CSS put the hidden one first, and the click waited on it.
+  return firstOf(visibleOnly(page.getByText(value, { exact: false })));
+}
+
+function firstOf(locator) {
+  return locator && typeof locator.first === 'function' ? locator.first() : locator;
 }
 
 /**
@@ -674,7 +1089,7 @@ function markDocument() {
  *
  * @param {{token: string, runId: string, jobId: string, persona: string}} identity
  */
-function injectPlaytestIdentity(identity) {
+export function injectPlaytestIdentity(identity) {
   window.__RAVENSIGHT_PLAYTEST__ = identity;
   try {
     localStorage.setItem('ravensight.playtest', JSON.stringify(identity));
@@ -684,6 +1099,15 @@ function injectPlaytestIdentity(identity) {
   }
 }
 
+/**
+ * The identity `check` hands a page it loads only to measure. The token is
+ * not a real run's, so a build shipping the Ravensight web SDK presents it on
+ * `POST /session` and the server refuses the session outright (a playtest
+ * claim that does not check out is a 403, never a real session): `check`
+ * is never recorded as a player.
+ */
+export const CHECK_PLAYTEST_IDENTITY = Object.freeze({ token: 'check', runId: null, jobId: null, persona: 'check' });
+
 /**
  * @param {Object} opts
  * @param {(specifier: string) => Promise<any>} [opts.importer] test seam
@@ -721,6 +1145,8 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
   // compared with the current one before any ref is used.
   let lastNodesDocument = null;
   const consoleBuffer = [];
+  /** Recent failed requests, newest last, bounded: what a console line is matched against. */
+  const networkFailures = [];
 
   /** @returns {any} */
   function requirePage() {
@@ -729,48 +1155,10 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
   }
 
   /**
-   * @returns {Promise<{text: string, aria: boolean}>} the screen text, and
-   *   whether it is the aria snapshot (compacted) or the innerText fallback
-   *   (never compacted: a bullet list in the page is not snapshot structure)
+   * @returns {Promise<{text: string, aria: boolean}>} see {@link readSnapshot}
    */
   async function snapshotText() {
-    const current = requirePage();
-    let text = '';
-    let aria = true;
-    const body = current.locator('body');
-    // The page level AI mode snapshot carries a ref per element, which
-    // `aria-ref=e12` resolves, and each element's box. Older Playwright
-    // builds have neither, and the body snapshot is the fallback.
-    if (typeof current.ariaSnapshot === 'function') {
-      try {
-        text = await current.ariaSnapshot({ mode: 'ai', boxes: true, timeout: ACTION_TIMEOUT_MS });
-      } catch {
-        text = '';
-      }
-    }
-    if (!text && typeof body.ariaSnapshot === 'function') {
-      try {
-        text = await body.ariaSnapshot({ timeout: ACTION_TIMEOUT_MS });
-      } catch {
-        text = '';
-      }
-    }
-    if (text.trim().length < THIN_OBSERVATION_CHARS) {
-      try {
-        const inner = await current.evaluate(() => {
-          const el = globalThis.document && globalThis.document.body;
-          return el ? el.innerText : '';
-        });
-        if (String(inner).trim().length > text.trim().length) {
-          text = String(inner);
-          aria = false;
-        }
-      } catch {
-        // A page mid-navigation has no body to read. The thin observation
-        // is the honest answer, and the screenshot fallback below covers it.
-      }
-    }
-    return { text, aria };
+    return readSnapshot(requirePage());
   }
 
   /**
@@ -783,7 +1171,7 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
    */
   function shownText(snap) {
     if (snap.aria) {
-      const fitted = fitAriaSnapshot(snap.text, { max: WEB_OBSERVATION_MAX_CHARS, previous: lastObservedLines ? new Set(lastObservedLines) : null, viewport: viewportSize });
+      const fitted = fitAriaSnapshot(snap.text, { max: WEB_OBSERVATION_MAX_CHARS, previous: lastObservedLines ? new Set(lastObservedLines) : null, viewport: viewportSize, values: snap.values });
       if (fitted) return fitted;
     }
     return { text: truncateObservation(snap.text, WEB_OBSERVATION_MAX_CHARS), lines: null, nodes: null };
@@ -818,11 +1206,19 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
    */
   async function typeText(current, action) {
     const text = String(action.text == null ? '' : action.text);
-    const field = action.target || action.ref ? await targetFor(current, action) : null;
+    let field = action.target || action.ref ? await targetFor(current, action) : null;
+    if (field && field.point) {
+      // Coordinates copied from the snapshot: focus what is there, then type
+      // into whatever took the focus.
+      await current.mouse.click(field.point.x, field.point.y);
+      field = null;
+    }
+    const subject = field || current.locator('*:focus');
+    const before = await fieldValue(subject);
     if (action.replace) {
       try {
-        await (field || current.locator('*:focus')).fill(text, { timeout: field ? ACTION_TIMEOUT_MS : FOCUSED_FILL_TIMEOUT_MS });
-        return;
+        await subject.fill(text, { timeout: field ? ACTION_TIMEOUT_MS : FOCUSED_FILL_TIMEOUT_MS });
+        return { before, after: await fieldValue(subject) };
       } catch {
         // Not something fill can write to. Focus it if it was named, then
         // clear it the way a player on this machine would.
@@ -834,29 +1230,51 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
       await field.click({ timeout: ACTION_TIMEOUT_MS });
     }
     if (text) await current.keyboard.type(text, { delay: 20 });
+    return { before, after: await fieldValue(subject) };
   }
 
   /**
-   * The locator a click or a typing target means. A ref is preferred and
-   * checked against the page; locator code is refused at once, with no wait;
-   * a name that matches more than one interactive node on the last screen is
-   * answered with the candidates and their refs rather than guessed.
+   * The ref locator, checked: a well formed ref, from the current document,
+   * and on the screen now.
+   */
+  async function refLocator(current, raw) {
+    const ref = String(raw).trim();
+    if (!REF_PATTERN.test(ref)) throw new TargetError(`a ref looks like e12, as the snapshot prints it; ${JSON.stringify(ref)} is not one`);
+    await assertSameDocument(current);
+    const locator = current.locator(`aria-ref=${ref}`);
+    if (typeof locator.count === 'function' && (await locator.count()) === 0) {
+      throw new TargetError(`ref ${ref} is not on the current screen; take a snapshot and use a ref from it`);
+    }
+    return locator;
+  }
+
+  /**
+   * The locator a click or a typing target means, or `{point}` for
+   * coordinates. A ref is preferred and checked against the page; a line
+   * pasted from the snapshot is read into its ref, else its coordinates
+   * (checked against the document like a ref); locator code is refused at once, with no wait; a name that
+   * matches more than one interactive node on the last screen is answered
+   * with the candidates and their refs rather than guessed, and a name that
+   * matches one is clicked by that node's ref. Anything else that is not on
+   * the last screen at all is refused at once rather than searched for.
    *
    * @param {any} current
    * @param {{ref?: string, target?: string}} action
    */
   async function targetFor(current, action) {
-    if (action.ref !== undefined && action.ref !== null && action.ref !== '') {
-      const ref = String(action.ref).trim();
-      if (!REF_PATTERN.test(ref)) throw new TargetError(`a ref looks like e12, as the snapshot prints it; ${JSON.stringify(ref)} is not one`);
+    if (action.ref !== undefined && action.ref !== null && action.ref !== '') return refLocator(current, action.ref);
+    const target = String(action.target == null ? '' : action.target);
+    let parsed = parseSnapshotTarget(target);
+    // A bare e4 that a node on screen is named exactly is that text (a chess
+    // square, a seat number), not a ref.
+    if (parsed && parsed.bare && Array.isArray(lastNodes) && lastNodes.some(node => node.name === target.trim() || node.body.endsWith(`: ${target.trim()}`))) parsed = null;
+    if (parsed) {
+      if (parsed.ref) return refLocator(current, parsed.ref);
+      // Coordinates read off a pasted line are as old as the screen they came
+      // from: refused like a ref when the document has changed since.
       await assertSameDocument(current);
-      const locator = current.locator(`aria-ref=${ref}`);
-      if (typeof locator.count === 'function' && (await locator.count()) === 0) {
-        throw new TargetError(`ref ${ref} is not on the current screen; take a snapshot and use a ref from it`);
-      }
-      return locator;
+      return { point: { x: parsed.x, y: parsed.y } };
     }
-    const target = String(action.target == null ? '' : action.target);
     const candidates = candidatesFor(lastNodes, target);
     if (candidates.length > 1) throw new TargetError(ambiguityMessage(target, candidates));
     if (LOCATOR_CODE.test(target)) {
@@ -868,6 +1286,16 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
       }
       throw new TargetError(LOCATOR_CODE_MESSAGE);
     }
+    const sameScreen = await isSameDocument(current);
+    // One node on screen carries that name: its ref names exactly it, where a
+    // text search could land on a hidden copy.
+    if (candidates.length === 1 && sameScreen) return current.locator(`aria-ref=${candidates[0].ref}`);
+    if (sameScreen && Array.isArray(lastNodes) && lastNodes.length > 0 && !KNOWN_SELECTOR_ENGINES.test(target.trim())) {
+      const roleAndName = target.trim().match(/^([a-z]+)\s+"([^"]*)"$/);
+      if (!textOnScreen(lastNodes, roleAndName ? roleAndName[2] : target)) {
+        throw new TargetError(`${JSON.stringify(target)} ${NOT_A_TARGET_MESSAGE}`);
+      }
+    }
     return resolveLocator(current, target);
   }
 
@@ -885,10 +1313,24 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
     }
   }
 
+  /** The page URL, or null when the page cannot answer. */
+  function pageUrl(current) {
+    try {
+      return typeof current.url === 'function' ? String(current.url()) : null;
+    } catch {
+      return null;
+    }
+  }
+
   /** Refuse a ref when the document changed since the observation it came from. */
   async function assertSameDocument(current) {
-    const now = await documentToken(current);
-    if (lastNodesDocument && now !== lastNodesDocument) throw new TargetError(STALE_REF_MESSAGE);
+    if (!(await isSameDocument(current))) throw new TargetError(STALE_REF_MESSAGE);
+  }
+
+  /** Whether the page is still the document the last observation came from. */
+  async function isSameDocument(current) {
+    if (!lastNodesDocument) return true;
+    return (await documentToken(current)) === lastNodesDocument;
   }
 
   /** The hash of the screen as it is now, on observe()'s basis. */
@@ -947,9 +1389,36 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
       if (typeof context.addInitScript === 'function') await context.addInitScript(markDocument);
       page = await context.newPage();
       page.on('console', message => {
-        if (message.type() === 'error') consoleBuffer.push({ type: 'console', text: message.text() });
+        if (message.type() !== 'error') return;
+        let location = null;
+        try {
+          location = typeof message.location === 'function' ? message.location() : null;
+        } catch {
+          location = null;
+        }
+        consoleBuffer.push({ type: 'console', text: message.text(), ...(location && location.url ? { locationUrl: String(location.url) } : {}) });
       });
       page.on('pageerror', error => consoleBuffer.push({ type: 'pageerror', text: String(error && error.message) }));
+      // The failing request behind a "Failed to load resource" line, which
+      // the console text alone never names.
+      page.on('response', response => {
+        try {
+          const status = response.status();
+          if (status < 400) return;
+          const request = typeof response.request === 'function' ? response.request() : null;
+          rememberNetworkFailure(networkFailures, { url: response.url(), status, method: request && typeof request.method === 'function' ? request.method() : null });
+        } catch {
+          // A response that cannot be read is not worth a note.
+        }
+      });
+      page.on('requestfailed', request => {
+        try {
+          const failure = typeof request.failure === 'function' ? request.failure() : null;
+          rememberNetworkFailure(networkFailures, { url: request.url(), status: null, method: typeof request.method === 'function' ? request.method() : null, error: failure && failure.errorText ? String(failure.errorText) : null });
+        } catch {
+          // Same: nothing to record.
+        }
+      });
 
       const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
       // A 404 is not something to guess around: the build is not there, and
@@ -1018,13 +1487,16 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
     async act(action) {
       const current = requirePage();
       const before = lastHash;
+      const urlBefore = pageUrl(current);
+      const tokenBefore = await documentToken(current);
+      let typed = null;
       try {
         switch (action.kind) {
           case 'key':
             await current.keyboard.press(normalizeChord(action.key, editModifier), { timeout: ACTION_TIMEOUT_MS });
             break;
           case 'type':
-            await typeText(current, action);
+            typed = await typeText(current, action);
             if (action.submit) await current.keyboard.press('Enter');
             break;
           case 'click': {
@@ -1032,7 +1504,9 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
               await current.mouse.click(Number(action.target.x), Number(action.target.y));
               break;
             }
-            await (await targetFor(current, action)).click({ timeout: ACTION_TIMEOUT_MS });
+            const resolved = await targetFor(current, action);
+            if (resolved && resolved.point) await current.mouse.click(resolved.point.x, resolved.point.y);
+            else await resolved.click({ timeout: ACTION_TIMEOUT_MS });
             break;
           }
           case 'navigate': {
@@ -1083,11 +1557,22 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
         return { ok: false, changed: false, error: text };
       }
 
-      let changed = false;
-      try {
+      const measure = async () => {
+        const facts = { urlBefore, urlAfter: pageUrl(current), tokenBefore, tokenAfter: await documentToken(current), hashBefore: before, valueBefore: typed ? typed.before : null, valueAfter: typed ? typed.after : null };
         // The same basis observe() hashes, so the two are comparable.
         // Without it a long screen is always "changed".
-        changed = (await currentHash(current)) !== before;
+        facts.hashAfter = await currentHash(current);
+        return computeChanged(facts);
+      };
+      let changed = false;
+      try {
+        changed = await measure();
+        // A click or a submit can start a navigation that has not landed yet:
+        // look once more after a short pause before calling it a no-op.
+        if (!changed && SETTLE_MS > 0 && (action.kind === 'click' || action.kind === 'key' || (action.kind === 'type' && action.submit))) {
+          await new Promise(resolve => setTimeout(resolve, SETTLE_MS));
+          changed = await measure();
+        }
       } catch {
         changed = true;
       }
@@ -1110,7 +1595,7 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
 
     /** @returns {Promise<Array<{type: string, text: string}>>} */
     async consoleErrors() {
-      return consoleBuffer.splice(0, consoleBuffer.length);
+      return attachNetworkFailures(consoleBuffer.splice(0, consoleBuffer.length), networkFailures);
     },
 
     /** @returns {Promise<{videoPath: string|null}>} */
@@ -1143,6 +1628,72 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
   };
 }
 
+/** How many recent failed requests are kept to match console lines against. */
+export const MAX_NETWORK_FAILURES = 50;
+
+/** The longest request URL a note carries. */
+export const MAX_NETWORK_URL_CHARS = 200;
+
+/**
+ * A request URL as a note carries it: no query string and no fragment (they
+ * carry tokens and session ids as often as anything useful), and bounded.
+ *
+ * @param {string} url
+ * @returns {string}
+ */
+export function networkUrl(url) {
+  let value = String(url == null ? '' : url);
+  try {
+    const parsed = new URL(value);
+    value = `${parsed.origin === 'null' ? `${parsed.protocol}` : parsed.origin}${parsed.pathname}`;
+  } catch {
+    value = value.split(/[?#]/)[0];
+  }
+  return value.length > MAX_NETWORK_URL_CHARS ? `${value.slice(0, MAX_NETWORK_URL_CHARS - 3)}...` : value;
+}
+
+/**
+ * @param {Array<Object>} list
+ * @param {{url: string, status: number|null, method?: string|null, error?: string|null}} failure
+ */
+export function rememberNetworkFailure(list, failure) {
+  list.push({ url: networkUrl(failure.url), status: Number.isInteger(failure.status) ? failure.status : null, method: failure.method || null, error: failure.error || null, claimed: false });
+  while (list.length > MAX_NETWORK_FAILURES) list.shift();
+}
+
+/** The console line a browser prints for a failed request. */
+const FAILED_RESOURCE = /^Failed to load resource\b/i;
+
+/**
+ * Give each "Failed to load resource" console line the request it was about:
+ * the one at the line's own location when the browser gave one, otherwise the
+ * oldest unclaimed failure with the same status (or the same network error).
+ * The line's text gains the method, the URL and the status, so the note says
+ * what failed; `url` and `status` ride along for the report.
+ *
+ * @param {Array<{type: string, text: string, locationUrl?: string}>} entries
+ * @param {Array<Object>} failures the driver's recent failed requests
+ * @returns {Array<Object>}
+ */
+export function attachNetworkFailures(entries, failures) {
+  return entries.map(entry => {
+    const { locationUrl, ...rest } = entry;
+    if (!FAILED_RESOURCE.test(String(entry.text || ''))) return rest;
+    const status = Number((String(entry.text).match(/status of (\d{3})/) || [])[1]) || null;
+    const location = locationUrl ? networkUrl(locationUrl) : null;
+    let match = location ? failures.find(failure => !failure.claimed && failure.url === location) : null;
+    if (!match) {
+      match = failures.find(failure => !failure.claimed && (status ? failure.status === status : failure.status === null && failure.error && String(entry.text).includes(failure.error)));
+    }
+    const url = match ? match.url : location;
+    if (!url) return rest;
+    if (match) match.claimed = true;
+    const shownStatus = match && match.status ? match.status : status;
+    const method = match && match.method ? `${match.method} ` : '';
+    return { ...rest, text: `${rest.text} (${method}${url}${shownStatus ? ` answered ${shownStatus}` : ''})`, url, status: shownStatus || null };
+  });
+}
+
 /**
  * Playwright names its recording with a random id. The upload allowlist and
  * the server's 90 day lifecycle rule are written against `session.webm`, so