ravensight-playtest 0.2.7 → 0.2.9

package/src/run/drivers/web.js

@@ -35,6 +35,12 @@ export const MAX_WAIT_MS = 10000;
 /** How long a single Playwright call may take before it is an error, not a hang. */
 export const ACTION_TIMEOUT_MS = 15000;
 
+/**
+ * How long a replace waits for a focused field. Short, because nothing
+ * focused is an ordinary answer (a canvas game), not a page still loading.
+ */
+export const FOCUSED_FILL_TIMEOUT_MS = 2000;
+
 /**
  * The web observation ceiling. An aria snapshot is re-sent on every turn the
  * observation stays in context, and the run's model budget is fixed, so a busy
@@ -71,9 +77,22 @@ const INTERACTIVE_ROLES = new Set([
  * @returns {string}
  */
 export function compactAriaSnapshot(text) {
+  const nodes = compactAriaNodes(text);
+  return nodes === null ? String(text == null ? '' : text) : nodes.map(node => node.line).join('\n');
+}
+
+/**
+ * The kept nodes of an aria snapshot with the depth each one had, or null for
+ * text that is not an aria snapshot. The depth is not shown to the model; it
+ * is what tells repeated siblings apart from a panel that follows them.
+ *
+ * @param {string} text
+ * @returns {Array<{depth: number, line: string, body: string, role: string}>|null}
+ */
+export function compactAriaNodes(text) {
   const value = String(text == null ? '' : text);
   const lines = value.split('\n');
-  if (!lines.some(line => /^\s*- /.test(line))) return value;
+  if (!lines.some(line => /^\s*- /.test(line))) return null;
   const kept = [];
   for (const line of lines) {
     const trimmed = line.trim();
@@ -83,9 +102,163 @@ export function compactAriaSnapshot(text) {
     const role = (body.match(/^([a-z]+)/) || [])[1] || '';
     const named = /^[a-z]+ "[^"]*"/.test(body);
     const inlineText = /^[a-z]+(?: "[^"]*")?(?: \[[^\]]*\])*: \S/.test(body);
-    if (INTERACTIVE_ROLES.has(role) || named || inlineText) kept.push(`- ${body}`);
+    if (INTERACTIVE_ROLES.has(role) || named || inlineText) {
+      kept.push({ depth: line.length - line.trimStart().length, line: `- ${body}`, body, role });
+    }
+  }
+  return kept;
+}
+
+/** Roles a player types into or sets, kept first when an observation has to be cut. */
+const INPUT_ROLES = new Set(['textbox', 'searchbox', 'combobox', 'checkbox', 'radio', 'slider', 'spinbutton', 'switch', 'listbox']);
+
+/** The share of interactive nodes, counted from the end of the page, kept ahead of the middle when a page is cut. */
+export const TAIL_FRACTION = 0.1;
+
+/** The tail window is never smaller than this many interactive nodes. */
+const TAIL_MIN = 3;
+
+/** A run of this many look-alike siblings or more is collapsed when the page is over the cap. */
+export const COLLAPSE_MIN_RUN = 4;
+
+/** How many of a collapsed run are still shown, so the model sees what the items look like. */
+export const COLLAPSE_KEEP = 2;
+
+/**
+ * The shape of one node, without the words that make it unique: its role,
+ * its state attributes, whether it has a name, whether the name carries
+ * digits and how many separators. Two school cards share a shape; the card
+ * list and a "TAKE THE JOB" button after it do not.
+ *
+ * @param {string} body
+ * @returns {string}
+ */
+function nodeShape(body) {
+  const role = (body.match(/^([a-z]+)/) || [])[1] || '';
+  const name = (body.match(/^[a-z]+ "([^"]*)"/) || [])[1];
+  const attributes = [...body.matchAll(/\[([a-z-]+)(?:=[^\]]*)?\]/g)].map(match => match[1]).filter(key => key !== 'ref').join(',');
+  const nameShape = name === undefined ? '' : `n${/\d/.test(name) ? 'd' : ''}${(name.match(/[·•|/]/g) || []).length}`;
+  const inline = /: \S/.test(body.replace(/^[a-z]+(?: "[^"]*")?/, '')) ? 't' : '';
+  return `${role}|${nameShape}|${attributes}|${inline}`;
+}
+
+/**
+ * Collapse runs of look-alike siblings (a node and its kept descendants,
+ * compared by shape) into the first {@link COLLAPSE_KEEP} plus one line that
+ * says how many more there are.
+ *
+ * @param {Array<{depth: number, line: string, body: string, role: string}>} nodes
+ * @returns {{entries: Array<{line: string, role: string, marker?: boolean}>, collapsed: number}}
+ */
+export function collapseRepeatedSiblings(nodes) {
+  const unitEnd = start => {
+    let end = start + 1;
+    while (end < nodes.length && nodes[end].depth > nodes[start].depth) end += 1;
+    return end;
+  };
+  const signature = (start, end) => nodes.slice(start, end).map(node => `${node.depth - nodes[start].depth}:${nodeShape(node.body)}`).join(';');
+  const entries = [];
+  let collapsed = 0;
+  let i = 0;
+  while (i < nodes.length) {
+    const units = [[i, unitEnd(i)]];
+    const sig = signature(i, units[0][1]);
+    let next = units[0][1];
+    while (next < nodes.length && nodes[next].depth === nodes[i].depth) {
+      const end = unitEnd(next);
+      if (signature(next, end) !== sig) break;
+      units.push([next, end]);
+      next = end;
+    }
+    if (units.length >= COLLAPSE_MIN_RUN) {
+      for (const [start, end] of units.slice(0, COLLAPSE_KEEP)) {
+        for (const node of nodes.slice(start, end)) entries.push({ line: node.line, role: node.role });
+      }
+      const more = units.length - COLLAPSE_KEEP;
+      collapsed += more;
+      entries.push({ line: `- ... and ${more} more like it (${nodes[i].role || 'items'}; target one by its name)`, role: '', marker: true });
+      i = next;
+    } else {
+      entries.push({ line: nodes[i].line, role: nodes[i].role });
+      i += 1;
+    }
+  }
+  return { entries, collapsed };
+}
+
+/**
+ * Fit a compacted aria snapshot under the web ceiling without losing the
+ * part of the page the persona has to act on.
+ *
+ * Under the cap nothing changes. Over it, in this order: interactive nodes
+ * are what is kept (the last tenth of them on the page first, since that is
+ * where a form after a long list sits, then inputs, then buttons and links,
+ * then any role with a name); runs of look-alike siblings collapse to their
+ * first two and a count; nodes that are new since the previous observation
+ * come next; then
+ * the rest, and only what still does not fit is trimmed. Kept lines stay in
+ * page order, and a first line says how many items were collapsed and how
+ * many lines were left out, so the model knows the screen was cut.
+ *
+ * @param {string} text the raw aria snapshot
+ * @param {{max?: number, previous?: Set<string>|null}} [options]
+ * @returns {{text: string, lines: Array<string>}|null} null when the text is not an aria snapshot
+ */
+export function fitAriaSnapshot(text, { max = WEB_OBSERVATION_MAX_CHARS, previous = null } = {}) {
+  const nodes = compactAriaNodes(text);
+  if (nodes === null) return null;
+  const lines = nodes.map(node => node.line);
+  const full = lines.join('\n');
+  if (full.length <= max) return { text: full, lines };
+
+  const { entries, collapsed } = collapseRepeatedSiblings(nodes);
+  const header = (left, out) => {
+    const parts = [];
+    if (left > 0) parts.push(`${left} similar items collapsed`);
+    if (out > 0) parts.push(`${out} lines left out to fit`);
+    return parts.length > 0 ? `[${parts.join('; ')}]` : '';
+  };
+  const collapsedText = entries.map(entry => entry.line).join('\n');
+  const withHeader = `${header(collapsed, 0)}\n${collapsedText}`;
+  if (withHeader.length <= max) return { text: withHeader, lines };
+
+  const rank = entry => {
+    if (entry.marker) return 0;
+    if (INPUT_ROLES.has(entry.role)) return 1;
+    if (entry.role === 'button' || entry.role === 'link') return 2;
+    if (INTERACTIVE_ROLES.has(entry.role) || /^- [a-z]+ "/.test(entry.line)) return 3;
+    return 5;
+  };
+  const isNew = entry => Boolean(previous) && !entry.marker && !previous.has(entry.line);
+  // The last interactive nodes on the page are usually its form or its
+  // confirm button (a panel after a list). When the list could not be
+  // collapsed they would otherwise lose to the list on page order, so the
+  // tail window, the final tenth of interactive nodes, ranks above the middle.
+  const interactive = entries.map((entry, index) => ({ entry, index })).filter(({ entry }) => !entry.marker && rank(entry) < 5);
+  const tailSize = Math.max(TAIL_MIN, Math.ceil(interactive.length * TAIL_FRACTION));
+  const tail = new Set(interactive.slice(-tailSize).map(({ index }) => index));
+  const tierOf = (entry, index) => {
+    if (entry.marker) return 0;
+    if (tail.has(index)) return 1 + (isNew(entry) ? 0 : 0.5);
+    if (rank(entry) < 5) return 2 + rank(entry) * 2 + (isNew(entry) ? 0 : 1);
+    return isNew(entry) ? 20 : 21;
+  };
+  const order = entries
+    .map((entry, index) => ({ entry, index, tier: tierOf(entry, index) }))
+    .sort((a, b) => a.tier - b.tier || a.index - b.index);
+  // Room for the header line, whose figures are not known until the end.
+  const budget = max - 80;
+  const chosen = new Set();
+  let used = 0;
+  for (const candidate of order) {
+    const cost = candidate.entry.line.length + 1;
+    if (used + cost > budget) continue;
+    chosen.add(candidate.index);
+    used += cost;
   }
-  return kept.join('\n');
+  const kept = entries.filter((entry, index) => chosen.has(index)).map(entry => entry.line);
+  const out = entries.length - kept.length;
+  return { text: truncateObservation(`${header(collapsed, out)}\n${kept.join('\n')}`, max), lines };
 }
 
 /**
@@ -116,6 +289,74 @@ export function unquoteKey(body) {
   return body;
 }
 
+/**
+ * The modifier a platform's editing shortcuts use: Command on macOS,
+ * Control everywhere else. Chromium follows the host's conventions even
+ * headless, so on a Mac `Control+A` in a text field moves the caret to the
+ * start of the line instead of selecting the text, and a persona that
+ * "selects all and retypes" ends up with two values joined together.
+ *
+ * @param {string} [platform] a `process.platform` value
+ * @returns {'Meta'|'Control'}
+ */
+export function platformModifier(platform = process.platform) {
+  return platform === 'darwin' ? 'Meta' : 'Control';
+}
+
+/** Editing shortcuts (select all, copy, cut, paste, undo, redo) that follow the platform modifier. */
+const EDITING_KEYS = new Set(['a', 'c', 'v', 'x', 'z', 'y']);
+
+/** Spellings a model writes for a modifier, mapped to Playwright's names. */
+const MODIFIER_ALIASES = Object.freeze({
+  ctrl: 'Control',
+  control: 'Control',
+  cmd: 'Meta',
+  command: 'Meta',
+  meta: 'Meta',
+  super: 'Meta',
+  win: 'Meta',
+  mod: 'ControlOrMeta',
+  controlormeta: 'ControlOrMeta',
+  alt: 'Alt',
+  option: 'Alt',
+  opt: 'Alt',
+  shift: 'Shift'
+});
+
+/**
+ * Rewrite a key chord the persona asked for into one that does the same
+ * thing on this machine.
+ *
+ * Modifier spellings are normalised (`Ctrl`, `Cmd`, `Option`), and an
+ * editing shortcut held with Control or Command (`Control+A`, `Cmd+V`,
+ * `Ctrl+Shift+Z`) uses the platform modifier, because the persona means
+ * "select all", not "the Control key". Any other chord is left as written:
+ * a game that binds Control+S to something of its own wants Control on
+ * every platform.
+ *
+ * @param {string} key
+ * @param {'Meta'|'Control'} modifier the platform modifier
+ * @returns {string}
+ */
+export function normalizeChord(key, modifier) {
+  const value = String(key == null ? '' : key);
+  // A lone "+" is the plus key, and "Shift++" is Shift and plus.
+  const parts = value.length > 1 ? value.split(/\+(?=.)/) : [value];
+  if (parts.length < 2) return value;
+  const mapped = parts.map((part, index) => {
+    if (index === parts.length - 1) return part;
+    return MODIFIER_ALIASES[part.trim().toLowerCase()] || part;
+  });
+  const last = mapped[mapped.length - 1];
+  const editing = EDITING_KEYS.has(String(last).toLowerCase());
+  return mapped.map((part, index) => {
+    if (index === mapped.length - 1) return part;
+    if (part === 'ControlOrMeta') return modifier;
+    if (editing && (part === 'Control' || part === 'Meta')) return modifier;
+    return part;
+  }).join('+');
+}
+
 const KNOWN_SELECTOR_ENGINES = /^(role|text|css|xpath|id|data-testid|internal:)=/;
 
 /**
@@ -231,9 +472,14 @@ function injectPlaytestIdentity(identity) {
  * @param {{token?: string, runId?: string, jobId?: string, persona?: string}} [opts.playtest]
  *   the run's playtest identity, handed to the page so the game's own SDK
  *   session is marked synthetic
+ * @param {string} [opts.platform] the host platform, `process.platform` by
+ *   default; decides the editing modifier
+ * @param {'Meta'|'Control'} [opts.modifier] the editing modifier, overriding
+ *   the one the platform implies
  * @returns {Object} a driver
  */
-export function createWebDriver({ importer, headless = true, playtest = null } = {}) {
+export function createWebDriver({ importer, headless = true, playtest = null, platform = process.platform, modifier = null } = {}) {
+  const editModifier = modifier === 'Meta' || modifier === 'Control' ? modifier : platformModifier(platform);
   let browser = null;
   let context = null;
   let page = null;
@@ -244,6 +490,8 @@ export function createWebDriver({ importer, headless = true, playtest = null } =
   let shots = 0;
   let lastHash = null;
   let lastObservedHash = null;
+  /** The compacted lines of the last observation, so a cut can prefer what is new. */
+  let lastObservedLines = null;
   const consoleBuffer = [];
 
   /** @returns {any} */
@@ -288,17 +536,71 @@ export function createWebDriver({ importer, headless = true, playtest = null } =
   }
 
   /**
-   * What the model is shown for a raw snapshot. `observe` and `act` both use
-   * this, so the two hashes compare like with like.
+   * What the model is shown for a raw snapshot, and the lines it was built
+   * from. A page over the web ceiling is fitted rather than cut at the
+   * character limit (see fitAriaSnapshot), so a panel after a long list
+   * still reaches the model.
    * @param {{text: string, aria: boolean}} snap
-   * @returns {string}
+   * @returns {{text: string, lines: Array<string>|null}}
    */
   function shownText(snap) {
-    return truncateObservation(snap.aria ? compactAriaSnapshot(snap.text) : snap.text, WEB_OBSERVATION_MAX_CHARS);
+    if (snap.aria) {
+      const fitted = fitAriaSnapshot(snap.text, { max: WEB_OBSERVATION_MAX_CHARS, previous: lastObservedLines ? new Set(lastObservedLines) : null });
+      if (fitted) return fitted;
+    }
+    return { text: truncateObservation(snap.text, WEB_OBSERVATION_MAX_CHARS), lines: null };
+  }
+
+  /**
+   * What `changed` and `unchanged` compare. For an aria snapshot that is the
+   * whole compacted tree, not the fitted text: the fit depends on the
+   * previous observation, so hashing it would read the same screen as
+   * changed. For the innerText fallback it is the truncated text, so a
+   * difference the model never saw is not a change.
+   * @param {{text: string, aria: boolean}} snap
+   * @param {{text: string, lines: Array<string>|null}} shown
+   * @returns {string}
+   */
+  function hashText(snap, shown) {
+    return shown.lines ? shown.lines.join('\n') : shown.text;
+  }
+
+  /**
+   * Type, or with `replace` put this text in place of what the field holds.
+   *
+   * Replacing goes through Playwright's `fill`, which clears the field
+   * natively on every platform, rather than a select-all chord: the chord is
+   * Command+A on a Mac and Control+A elsewhere, and getting it wrong joins the
+   * old value to the new one. A target names the field; without one the
+   * focused element is filled. Only an element `fill` refuses (a canvas, a
+   * custom widget) falls back to the platform select-all, then typing.
+   *
+   * @param {any} current the page
+   * @param {{text: string, target?: string, replace?: boolean}} action
+   */
+  async function typeText(current, action) {
+    const text = String(action.text == null ? '' : action.text);
+    const field = action.target ? resolveLocator(current, action.target) : null;
+    if (action.replace) {
+      try {
+        await (field || current.locator('*:focus')).fill(text, { timeout: field ? ACTION_TIMEOUT_MS : FOCUSED_FILL_TIMEOUT_MS });
+        return;
+      } catch {
+        // Not something fill can write to. Focus it if it was named, then
+        // clear it the way a player on this machine would.
+        if (field) await field.click({ timeout: ACTION_TIMEOUT_MS });
+        await current.keyboard.press(`${editModifier}+a`);
+        await current.keyboard.press('Backspace');
+      }
+    } else if (field) {
+      await field.click({ timeout: ACTION_TIMEOUT_MS });
+    }
+    if (text) await current.keyboard.type(text, { delay: 20 });
   }
 
   return {
     name: 'playwright_web',
+    editModifier,
     keepObservations: WEB_KEEP_OBSERVATIONS,
 
     /**
@@ -355,9 +657,10 @@ export function createWebDriver({ importer, headless = true, playtest = null } =
       step += 1;
       const snap = await snapshotText();
       const text = snap.text;
+      const shown = shownText(snap);
       const observation = {
         step,
-        text: shownText(snap),
+        text: shown.text,
         url: current.url(),
         elapsedMs: Date.now() - started
       };
@@ -384,11 +687,11 @@ export function createWebDriver({ importer, headless = true, playtest = null } =
           // read it as a blank screen, which is what it is.
         }
       }
-      // Hash what was REPORTED, not the untruncated snapshot: `act` compares
-      // against this, so two screens that differ only past the truncation
-      // point would otherwise read as changed to `act` and identical to
-      // anything reading the observation.
-      lastHash = observationHash(observation);
+      // Hash what hashText says, which `act` hashes the same way: the whole
+      // compacted tree for an aria snapshot (a change anywhere in it is shown,
+      // because a fit keeps new nodes first), the truncated text otherwise.
+      lastHash = observationHash({ text: hashText(snap, shown), screenshotBase64: observation.screenshotBase64 });
+      lastObservedLines = shown.lines;
       // The same screen twice is sent as a marker rather than in full
       // (./tools.js), which is most of what a stuck persona would otherwise
       // pay for.
@@ -407,10 +710,10 @@ export function createWebDriver({ importer, headless = true, playtest = null } =
       try {
         switch (action.kind) {
           case 'key':
-            await current.keyboard.press(String(action.key), { timeout: ACTION_TIMEOUT_MS });
+            await current.keyboard.press(normalizeChord(action.key, editModifier), { timeout: ACTION_TIMEOUT_MS });
             break;
           case 'type':
-            await current.keyboard.type(String(action.text), { delay: 20 });
+            await typeText(current, action);
             if (action.submit) await current.keyboard.press('Enter');
             break;
           case 'click': {
@@ -441,10 +744,10 @@ export function createWebDriver({ importer, headless = true, playtest = null } =
 
       let changed = false;
       try {
-        // The same truncation observe() applies, so the two hashes are
-        // comparable. Without it a long screen is always "changed".
+        // The same basis observe() hashes, so the two are comparable.
+        // Without it a long screen is always "changed".
         const snap = await snapshotText();
-        const after = { text: shownText(snap) };
+        const after = { text: hashText(snap, shownText(snap)) };
         if (String(snap.text).trim().length < THIN_OBSERVATION_CHARS) {
           const frame = await current.screenshot({ type: 'png', timeout: ACTION_TIMEOUT_MS });
           after.screenshotBase64 = frame.toString('base64');

package/src/run/drivers/webdriver.js

@@ -51,6 +51,7 @@ export function createWebDriverAdapter({name='desktop_driver',fetch:fetchImpl=gl
     }else if(action.kind==='swipe'){
      const from=point(action.from),to=point(action.to);await actions('pointer',[{type:'pointerMove',duration:0,origin:'viewport',...from},{type:'pointerDown',button:0},{type:'pointerMove',duration:Math.min(2000,Math.max(100,Number(action.durationMs)||500)),origin:'viewport',...to},{type:'pointerUp',button:0}],touch);
     }else if(action.kind==='key'||action.kind==='type'){
+     if(action.kind==='type'&&(action.replace||action.target))throw new DriverError('replace (empty a field, then type) and a typing target are only available on playwright_web; this driver can only type more text into what has focus');
      const chars=action.kind==='type'?Array.from(String(action.text||'').slice(0,2000)):[KEYS[action.key]||action.key];
      if(chars.some(key=>typeof key!=='string'||Array.from(key).length!==1))throw new DriverError('Unsupported device key.');
      if(action.submit)chars.push(KEYS.Enter);