ravensight-playtest 0.2.3 → 0.2.5

package/src/run/drivers/web.js

@@ -2,7 +2,6 @@ import { mkdir, rename, stat, readdir } from 'node:fs/promises';
 import path from 'node:path';
 import {
   DriverError,
-  OBSERVATION_MAX_CHARS,
   THIN_OBSERVATION_CHARS,
   observationHash,
   screenshotName,
@@ -36,6 +35,87 @@ export const MAX_WAIT_MS = 10000;
 /** How long a single Playwright call may take before it is an error, not a hang. */
 export const ACTION_TIMEOUT_MS = 15000;
 
+/**
+ * The web observation ceiling. An aria snapshot is re-sent on every turn the
+ * observation stays in context, and the run's model budget is fixed, so a busy
+ * page at the shared 12,000 character ceiling spends the budget on markup the
+ * persona never needed. Compaction (below) usually lands well under this.
+ */
+export const WEB_OBSERVATION_MAX_CHARS = 4000;
+
+/**
+ * Full observations kept in context for a web run. The newest screen is what
+ * the persona acts on; one more is enough to compare against, and every older
+ * one is a stub.
+ */
+export const WEB_KEEP_OBSERVATIONS = 2;
+
+/** Roles a player can act on. Kept even with no name, since they are still clickable. */
+const INTERACTIVE_ROLES = new Set([
+  'button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'combobox', 'listbox', 'option',
+  'slider', 'spinbutton', 'switch', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'treeitem',
+  'canvas'
+]);
+
+/**
+ * Keep the nodes a player could act on or read, and drop the structure in
+ * between. An aria snapshot prints every wrapper (`- generic:`, `- list:`, a
+ * nameless `- img`) and the attribute lines under links (`- /url: ...`), none
+ * of which tell a persona anything, and all of which cost tokens every turn.
+ *
+ * Indentation is dropped too: the persona targets nodes by role and name, not
+ * by depth. Text that is not an aria snapshot (the innerText fallback for a
+ * thin page) is returned unchanged.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+export function compactAriaSnapshot(text) {
+  const value = String(text == null ? '' : text);
+  const lines = value.split('\n');
+  if (!lines.some(line => /^\s*- /.test(line))) return value;
+  const kept = [];
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed.startsWith('- ')) continue;
+    const body = unquoteKey(trimmed.slice(2));
+    if (body.startsWith('/')) continue;
+    const role = (body.match(/^([a-z]+)/) || [])[1] || '';
+    const named = /^[a-z]+ "[^"]*"/.test(body);
+    const inlineText = /^[a-z]+(?: "[^"]*")?(?: \[[^\]]*\])*: \S/.test(body);
+    if (INTERACTIVE_ROLES.has(role) || named || inlineText) kept.push(`- ${body}`);
+  }
+  return kept.join('\n');
+}
+
+/**
+ * Playwright writes a node key as a single quoted YAML scalar when the name
+ * contains `: `, ` #`, a brace or a backtick: `- 'button "Player #1"'`. Strip
+ * that outer pair (a doubled `''` inside is one quote) so the node is
+ * classified and shown like any other, keeping whatever followed the closing
+ * quote (a `:` for a node with children, or `: text`).
+ *
+ * @param {string} body the line after its `- `
+ * @returns {string}
+ */
+export function unquoteKey(body) {
+  if (!body.startsWith("'")) return body;
+  let out = '';
+  for (let i = 1; i < body.length; i += 1) {
+    if (body[i] === "'") {
+      if (body[i + 1] === "'") {
+        out += "'";
+        i += 1;
+        continue;
+      }
+      return out + body.slice(i + 1);
+    }
+    out += body[i];
+  }
+  // No closing quote: not a quoted key after all.
+  return body;
+}
+
 const KNOWN_SELECTOR_ENGINES = /^(role|text|css|xpath|id|data-testid|internal:)=/;
 
 /**
@@ -127,12 +207,33 @@ export function resolveLocator(page, target) {
   return page.getByText(value, { exact: false }).first();
 }
 
+/**
+ * Hand the run's playtest identity to the page. Runs in the browser, before
+ * any of the game's own scripts, so the Ravensight JS SDK finds it at
+ * construction. Serialised by Playwright, so it must not close over anything.
+ *
+ * @param {{token: string, runId: string, jobId: string, persona: string}} identity
+ */
+function injectPlaytestIdentity(identity) {
+  window.__RAVENSIGHT_PLAYTEST__ = identity;
+  try {
+    localStorage.setItem('ravensight.playtest', JSON.stringify(identity));
+  } catch {
+    // Storage can be refused (a sandboxed frame, a privacy mode). The window
+    // global is the first thing the SDK reads, so this is only the fallback.
+  }
+}
+
 /**
  * @param {Object} opts
  * @param {(specifier: string) => Promise<any>} [opts.importer] test seam
+ * @param {boolean} [opts.headless]
+ * @param {{token?: string, runId?: string, jobId?: string, persona?: string}} [opts.playtest]
+ *   the run's playtest identity, handed to the page so the game's own SDK
+ *   session is marked synthetic
  * @returns {Object} a driver
  */
-export function createWebDriver({ importer, headless = true } = {}) {
+export function createWebDriver({ importer, headless = true, playtest = null } = {}) {
   let browser = null;
   let context = null;
   let page = null;
@@ -142,6 +243,7 @@ export function createWebDriver({ importer, headless = true } = {}) {
   let step = 0;
   let shots = 0;
   let lastHash = null;
+  let lastObservedHash = null;
   const consoleBuffer = [];
 
   /** @returns {any} */
@@ -150,9 +252,15 @@ export function createWebDriver({ importer, headless = true } = {}) {
     return page;
   }
 
+  /**
+   * @returns {Promise<{text: string, aria: boolean}>} the screen text, and
+   *   whether it is the aria snapshot (compacted) or the innerText fallback
+   *   (never compacted: a bullet list in the page is not snapshot structure)
+   */
   async function snapshotText() {
     const current = requirePage();
     let text = '';
+    let aria = true;
     const body = current.locator('body');
     if (typeof body.ariaSnapshot === 'function') {
       try {
@@ -167,17 +275,31 @@ export function createWebDriver({ importer, headless = true } = {}) {
           const el = globalThis.document && globalThis.document.body;
           return el ? el.innerText : '';
         });
-        if (String(inner).trim().length > text.trim().length) text = String(inner);
+        if (String(inner).trim().length > text.trim().length) {
+          text = String(inner);
+          aria = false;
+        }
       } catch {
         // A page mid-navigation has no body to read. The thin observation
         // is the honest answer, and the screenshot fallback below covers it.
       }
     }
-    return text;
+    return { text, aria };
+  }
+
+  /**
+   * What the model is shown for a raw snapshot. `observe` and `act` both use
+   * this, so the two hashes compare like with like.
+   * @param {{text: string, aria: boolean}} snap
+   * @returns {string}
+   */
+  function shownText(snap) {
+    return truncateObservation(snap.aria ? compactAriaSnapshot(snap.text) : snap.text, WEB_OBSERVATION_MAX_CHARS);
   }
 
   return {
     name: 'playwright_web',
+    keepObservations: WEB_KEEP_OBSERVATIONS,
 
     /**
      * @param {Object} opts
@@ -200,6 +322,18 @@ export function createWebDriver({ importer, headless = true } = {}) {
         viewport,
         ...(videoRequested ? { recordVideo: { dir: runDir, size: viewport } } : {})
       });
+      // The playtest identity goes in as an init script, not as
+      // `extraHTTPHeaders`: a context header rides on every request the page
+      // makes, including to the game's own API, and the token is only for
+      // Ravensight. The SDK reads it from the page and sends it itself.
+      if (playtest && playtest.token) {
+        await context.addInitScript(injectPlaytestIdentity, {
+          token: playtest.token,
+          runId: playtest.runId || null,
+          jobId: playtest.jobId || null,
+          persona: playtest.persona || null
+        });
+      }
       page = await context.newPage();
       page.on('console', message => {
         if (message.type() === 'error') consoleBuffer.push({ type: 'console', text: message.text() });
@@ -219,10 +353,11 @@ export function createWebDriver({ importer, headless = true } = {}) {
       const current = requirePage();
       const started = Date.now();
       step += 1;
-      const text = await snapshotText();
+      const snap = await snapshotText();
+      const text = snap.text;
       const observation = {
         step,
-        text: truncateObservation(text, OBSERVATION_MAX_CHARS),
+        text: shownText(snap),
         url: current.url(),
         elapsedMs: Date.now() - started
       };
@@ -244,6 +379,11 @@ export function createWebDriver({ importer, headless = true } = {}) {
       // point would otherwise read as changed to `act` and identical to
       // anything reading the observation.
       lastHash = observationHash(observation);
+      // The same screen twice is sent as a marker rather than in full
+      // (./tools.js), which is most of what a stuck persona would otherwise
+      // pay for.
+      if (lastHash === lastObservedHash) observation.unchanged = true;
+      lastObservedHash = lastHash;
       return observation;
     },
 
@@ -293,7 +433,13 @@ export function createWebDriver({ importer, headless = true } = {}) {
       try {
         // The same truncation observe() applies, so the two hashes are
         // comparable. Without it a long screen is always "changed".
-        changed = observationHash(truncateObservation(await snapshotText(), OBSERVATION_MAX_CHARS)) !== before;
+        const snap = await snapshotText();
+        const after = { text: shownText(snap) };
+        if (String(snap.text).trim().length < THIN_OBSERVATION_CHARS) {
+          const frame = await current.screenshot({ type: 'png', timeout: ACTION_TIMEOUT_MS });
+          after.screenshotBase64 = frame.toString('base64');
+        }
+        changed = observationHash(after) !== before;
       } catch {
         changed = true;
       }

package/src/run/drivers/webdriver.js

@@ -0,0 +1,67 @@
+// Native desktop and mobile adapters use the W3C session/actions protocol.
+// The developer supplies a local Appium/WebDriver server and a reviewed config;
+// game observations cannot choose servers, capabilities, executables or scripts.
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { DriverError, observationHash, screenshotName, truncateObservation } from './driver.js';
+
+export const WEBDRIVER_NAMES = ['desktop_driver', 'unity_driver', 'unreal_driver', 'android_driver', 'ios_driver'];
+const KEYS = { Enter:'\uE007', Escape:'\uE00C', Space:' ', Tab:'\uE004', Backspace:'\uE003', ArrowUp:'\uE013', ArrowDown:'\uE015', ArrowLeft:'\uE012', ArrowRight:'\uE014' };
+const MAX_BYTES = 8 * 1024 * 1024;
+export async function readDriverConfig(file) {
+  const stat=await fs.stat(file); if(stat.size>64000) throw new DriverError('Driver configuration exceeds 64 KB.',{fatal:true});
+  const config=JSON.parse(await fs.readFile(file,'utf8'));
+  const url=new URL(config.endpoint);
+  if(!['http:','https:'].includes(url.protocol)||!['localhost','127.0.0.1','[::1]'].includes(url.hostname)||url.username||url.password||url.search||url.hash) throw new DriverError('The device server must use a loopback URL without credentials.',{fatal:true});
+  if(!config.capabilities||Array.isArray(config.capabilities)||typeof config.capabilities!=='object'||typeof config.capabilities.platformName!=='string') throw new DriverError('Supply W3C capabilities with platformName.',{fatal:true});
+  return {...config,endpoint:url.href.replace(/\/$/,'')};
+}
+export function createWebDriverAdapter({name='desktop_driver',fetch:fetchImpl=globalThis.fetch,sleep=ms=>new Promise(resolve=>setTimeout(resolve,ms))}={}) {
+ let config,session,runDirectory,step=0,shots=0,lastHash=null;
+ async function request(method,route,body) {
+  const response=await fetchImpl(config.endpoint+route,{method,redirect:'error',signal:AbortSignal.timeout(30000),headers:{'Content-Type':'application/json'},...(body?{body:JSON.stringify(body)}:{})});
+  if(response.status===204) return null;
+  if(!response.body) throw new DriverError('Device response was empty.');
+  const reader=response.body.getReader();const chunks=[];let size=0;
+  try {for(;;){const part=await reader.read();if(part.done)break;size+=part.value.length;if(size>MAX_BYTES){await reader.cancel();throw new DriverError('Device response exceeded the limit.');}chunks.push(Buffer.from(part.value));}}finally{reader.releaseLock();}
+  const data=JSON.parse(Buffer.concat(chunks).toString());
+  if(!response.ok||data.value?.error)throw new DriverError('Device command failed: '+String(data.value?.error||response.status));
+  return data.value;
+ }
+ const route=tail=>{if(!session)throw new DriverError('No device session is active.',{fatal:true});return '/session/'+encodeURIComponent(session)+tail;};
+ async function frame(){const data=await request('GET',route('/screenshot'));if(typeof data!=='string'||!data.startsWith('iVBORw0KGgo'))throw new DriverError('Device screenshot is not a PNG.');return data;}
+ async function observe(){const [source,image]=await Promise.all([request('GET',route('/source')),frame()]);const observation={step:++step,text:truncateObservation(typeof source==='string'?source:JSON.stringify(source)),screenshotBase64:image,screenshotMediaType:'image/png',elapsedMs:0};lastHash=observationHash(observation);return observation;}
+ async function actions(type,values,pointerType='mouse'){await request('POST',route('/actions'),{actions:[{type,id:'ravensight',...(type==='pointer'?{parameters:{pointerType}}:{}),actions:values}]});}
+ const driver={name,supportedActions:['click','type','key','wait','swipe'],
+  async launch({target,runDir}){
+   config=await readDriverConfig(target);runDirectory=runDir;
+   const platform=config.capabilities.platformName.toLowerCase();
+   if(name==='android_driver'&&platform!=='android'||name==='ios_driver'&&platform!=='ios')throw new DriverError('The device platform does not match the selected adapter.',{fatal:true});
+   const answer=await request('POST','/session',{capabilities:{alwaysMatch:config.capabilities,firstMatch:[{}]}});
+   if(!answer||typeof answer.sessionId!=='string')throw new DriverError('The device server did not create a session.',{fatal:true});
+   session=answer.sessionId;await fs.mkdir(path.join(runDir,'screenshots'),{recursive:true});
+  },observe,
+  async act(action){
+   const before=lastHash;const touch=['android_driver','ios_driver'].includes(name)?'touch':'mouse';
+   const point=value=>{if(!value||!Number.isFinite(value.x)||!Number.isFinite(value.y)||value.x<0||value.y<0||value.x>32768||value.y>32768)throw new DriverError('Supply bounded screenshot coordinates.');return {x:Math.round(value.x),y:Math.round(value.y)};};
+   try {
+    if(action.kind==='wait')await sleep(Math.min(5000,Math.max(0,Number(action.ms)||0)));
+    else if(action.kind==='click'){
+     const p=point(action.target);await actions('pointer',[{type:'pointerMove',duration:0,origin:'viewport',...p},{type:'pointerDown',button:0},{type:'pointerUp',button:0}],touch);
+    }else if(action.kind==='swipe'){
+     const from=point(action.from),to=point(action.to);await actions('pointer',[{type:'pointerMove',duration:0,origin:'viewport',...from},{type:'pointerDown',button:0},{type:'pointerMove',duration:Math.min(2000,Math.max(100,Number(action.durationMs)||500)),origin:'viewport',...to},{type:'pointerUp',button:0}],touch);
+    }else if(action.kind==='key'||action.kind==='type'){
+     const chars=action.kind==='type'?Array.from(String(action.text||'').slice(0,2000)):[KEYS[action.key]||action.key];
+     if(chars.some(key=>typeof key!=='string'||Array.from(key).length!==1))throw new DriverError('Unsupported device key.');
+     if(action.submit)chars.push(KEYS.Enter);
+     await actions('key',chars.flatMap(value=>[{type:'keyDown',value},{type:'pause',duration:Math.min(2000,Math.max(0,Number(action.durationMs)||0))},{type:'keyUp',value}]));
+    }else throw new DriverError('This device adapter does not support that action.');
+    return {ok:true,changed:observationHash(await observe())!==before};
+   }catch(error){return {ok:false,changed:false,error:error instanceof DriverError?error.message:'Device input failed.'};}
+   finally {if(session)await request('DELETE',route('/actions')).catch(()=>{});}
+  },
+  async screenshot(reason){const relativePath=path.join('screenshots',screenshotName(++shots,reason));const file=path.join(runDirectory,relativePath);await fs.writeFile(file,Buffer.from(await frame(),'base64'));return {path:file,relativePath,reason};},
+  async consoleErrors(){return [];},
+  async stop(){if(session){const current=route('');session=null;await request('DELETE',current);}return {videoPath:null};}
+ };return driver;
+}

package/src/run/fixLoop.js

@@ -51,6 +51,55 @@ export const PLAN_MAX_TURNS = 6;
  */
 export const NUDGE_EVERY = 8;
 
+/** One breakpoint, frozen so no caller can hand a mutated one to the next turn. */
+const CACHE_BREAKPOINT = Object.freeze({ type: 'ephemeral' });
+
+/**
+ * Move a cache breakpoint onto the newest turn of a conversation.
+ *
+ * `fixSystem` already marks the end of the system prompt, but a breakpoint
+ * there only ever caches the skill and the repository facts. The tokens on
+ * this path are in `messages`: a `read_file` result is a whole source file,
+ * and the apply loop accumulates dozens of them over up to sixty turns, every
+ * one of which re-sends the whole conversation. Without a breakpoint in
+ * `messages` that conversation is processed at full price on every turn, which
+ * is quadratic in the number of turns and is where `fix.apply`'s bill comes
+ * from.
+ *
+ * The prefix written on turn N is exactly the prefix read on turn N+1, because
+ * these two loops only ever APPEND to `messages`. Nothing rewrites an earlier
+ * turn, so the bytes ahead of the marker are identical from one call to the
+ * next. (`personaLoop` deliberately does not do this: `prepareMessages`
+ * rewrites older observations into stubs as a run goes on, so its prefix is not
+ * stable and a breakpoint in its messages would pay the write premium instead
+ * of earning a read.)
+ *
+ * Applied to a COPY, leaving the stored conversation unmarked, for two
+ * reasons. A request may carry at most four breakpoints, so a marker left
+ * behind on each turn would be refused by the fifth. And the bytes this call
+ * sends for an earlier turn have to be the bytes the last call sent, which a
+ * marker written into the history would change. A marker that moves is not an
+ * invalidator; one that accumulates is.
+ *
+ * No breakpoint can leak anything across runs or across customers. A cache
+ * read requires the caller to present the identical prefix bytes, so the only
+ * thing an entry can ever serve back is a prompt whoever asked for it already
+ * held. The proxy passes `messages` through by reference, so nothing about
+ * this needs the server's field allowlist widened.
+ *
+ * @param {Array<Object>} messages the conversation so far, unmarked
+ * @returns {Array<Object>} the conversation to send, with one moving marker
+ */
+export function withRollingCacheBreakpoint(messages) {
+  const last = messages.at(-1);
+  if (!last || !Array.isArray(last.content) || last.content.length === 0) return messages;
+  const block = last.content.at(-1);
+  if (!block || typeof block !== 'object') return messages;
+  const content = [...last.content];
+  content[content.length - 1] = { ...block, cache_control: CACHE_BREAKPOINT };
+  return [...messages.slice(0, -1), { ...last, content }];
+}
+
 /** The text of a tool result, whichever of the two content shapes it uses. */
 function resultText(result) {
   if (!result) return '';
@@ -213,7 +262,7 @@ export async function planFix({
       runId,
       step: STEPS.fixPlan,
       system,
-      messages,
+      messages: withRollingCacheBreakpoint(messages),
       tools: repo ? repo.tools : undefined,
       maxTokens: PLAN_MAX_TOKENS
     });
@@ -337,7 +386,7 @@ export async function applyFix({
         runId,
         step: STEPS.fixApply,
         system,
-        messages,
+        messages: withRollingCacheBreakpoint(messages),
         tools: repo.tools,
         maxTokens: APPLY_MAX_TOKENS
       });
@@ -486,4 +535,4 @@ export function clamp(value, limit) {
   return text.length <= limit ? text : text.slice(0, limit);
 }
 
-export default { planFix, applyFix, summariseFix, resolveFixMaxActions, fixSystem, planContent, normalisePlan };
+export default { planFix, applyFix, summariseFix, resolveFixMaxActions, fixSystem, planContent, normalisePlan, withRollingCacheBreakpoint };