ravensight-playtest 0.2.9 → 0.3.1

package/src/run/regenerate.js

@@ -47,7 +47,7 @@ export async function readTranscript(runDirectory) {
   try {
     text = await readFile(path.join(runDirectory, RUN_FILES.transcript), 'utf8');
   } catch {
-    return { header: null, notes: [], screenshots: [], observations: 0, actions: 0 };
+    return { header: null, notes: [], screenshots: [], observations: 0, actions: 0, firstT: null, lastT: null };
   }
 
   let header = null;
@@ -55,6 +55,12 @@ export async function readTranscript(runDirectory) {
   const screenshots = [];
   let observations = 0;
   let actions = 0;
+  // Notes written on the tool call the last `action` line closed, so a
+  // `driver_error` after it can mark them as the live run did.
+  let callNotes = [];
+  let openNotes = [];
+  let firstT = null;
+  let lastT = null;
   for (const line of text.split('\n')) {
     if (!line.trim()) continue;
     let entry;
@@ -65,13 +71,27 @@ export async function readTranscript(runDirectory) {
       // it rather than refusing to read the whole transcript.
       continue;
     }
+    if (Number.isFinite(Number(entry.t))) {
+      if (firstT === null) firstT = Number(entry.t);
+      lastT = Number(entry.t);
+    }
     if (entry.kind === 'header') header = entry;
-    else if (entry.kind === 'note' && entry.note) notes.push(entry.note);
+    else if (entry.kind === 'note' && entry.note) {
+      const note = { ...entry.note };
+      notes.push(note);
+      if (entry.tool) openNotes.push(note);
+    } else if (entry.kind === 'driver_error') {
+      for (const note of callNotes) note.driver_error = true;
+    }
     else if (entry.kind === 'screenshot' && entry.path) screenshots.push({ relativePath: entry.path, reason: entry.reason || '' });
     else if (entry.kind === 'observation') observations += 1;
-    else if (entry.kind === 'action' && Number.isFinite(Number(entry.actions))) actions = Math.max(actions, Number(entry.actions));
+    else if (entry.kind === 'action') {
+      callNotes = openNotes;
+      openNotes = [];
+      if (Number.isFinite(Number(entry.actions))) actions = Math.max(actions, Number(entry.actions));
+    }
   }
-  return { header, notes, screenshots, observations, actions };
+  return { header, notes, screenshots, observations, actions, firstT, lastT };
 }
 
 /**

package/src/run/resume.js

@@ -244,7 +244,13 @@ async function finalizeFromTranscript({ deps, api, gameId, jobId, repoRoot, row,
         actions_taken: report.actions_taken,
         quit_reason: 'error',
         report,
-        report_md: markdown
+        report_md: markdown,
+        // The transcript's own span: its first and last timestamps. No model
+        // time is known for a run rebuilt after the process died.
+        duration: {
+          wall_seconds: transcript.firstT !== null && transcript.lastT !== null ? Number(((transcript.lastT - transcript.firstT) / 1000).toFixed(3)) : null,
+          model_seconds: null
+        }
       }
     });
     if (answer && answer.dropped_findings > 0) {

package/src/run/tools.js

@@ -1,5 +1,7 @@
 import { UNCHANGED_OBSERVATION_TEXT } from './context.js';
-import { OBSERVATION_MAX_CHARS, truncateObservation } from './drivers/driver.js';
+import { OBSERVATION_MAX_CHARS, STALE_REF_MESSAGE, truncateObservation } from './drivers/driver.js';
+
+export { STALE_REF_MESSAGE };
 
 /**
  * The client tool set for `persona_playtest.act`.
@@ -23,8 +25,12 @@ import { OBSERVATION_MAX_CHARS, truncateObservation } from './drivers/driver.js'
  * `snapshot`, `screenshot` and `wait` do not.
  */
 
-/** Note kinds, from spec 03's `log_note`. */
-export const NOTE_KINDS = Object.freeze(['bug', 'confusing', 'good', 'suggestion', 'reaction']);
+/**
+ * Note kinds, from spec 03's `log_note`, plus `waiting`: the persona is
+ * waiting for something to finish (a load, an animation, a turn). A waiting
+ * note is context for the report and never becomes a finding.
+ */
+export const NOTE_KINDS = Object.freeze(['bug', 'confusing', 'good', 'suggestion', 'reaction', 'waiting']);
 
 /** Severities, from the report schema's finding severity enum. */
 export const SEVERITIES = Object.freeze(['blocker', 'major', 'minor', 'cosmetic', 'praise']);
@@ -52,7 +58,7 @@ const note = {
   additionalProperties: false,
   required: ['kind', 'text'],
   properties: {
-    kind: { type: 'string', enum: [...NOTE_KINDS] },
+    kind: { type: 'string', enum: [...NOTE_KINDS], description: 'waiting means you are waiting for something to finish; it is recorded as context and never reported as a finding.' },
     text: { type: 'string' },
     severity: { type: 'string', enum: [...SEVERITIES] }
   }
@@ -62,7 +68,7 @@ const note = {
 export const PLAY_TOOLS = Object.freeze([
   {
     name: 'snapshot',
-    description: 'Look at the current screen. Does not spend an action. Use it after anything that might have changed the game state.',
+    description: 'Look at the current screen. Does not spend an action. Use it after anything that might have changed the game state. The result also repeats this run\'s identity (the email and display name to use whenever a form asks for one).',
     input_schema: {
       type: 'object',
       additionalProperties: false,
@@ -71,12 +77,13 @@ export const PLAY_TOOLS = Object.freeze([
   },
   {
     name: 'click',
-    description: 'Click one thing. Spends an action. Either name the target or give x and y pixel coordinates. What a named target means depends on the driver: on playwright_web it is an accessibility node such as button "Start", a visible text fragment, or a CSS selector; on godot_driver it is a node path from the scene tree you were shown.',
+    description: 'Click one thing. Spends an action. On playwright_web, use the ref the screen shows next to the element, such as e12: it names exactly one element, even when two share a name. Otherwise name the target: on playwright_web an accessibility node such as button "Start" or a visible text fragment; on godot_driver a node path from the scene tree you were shown. Targets are refs or visible text, never locator code such as getByRole(...) or page.locator(...). Avoid x and y coordinates on playwright_web: they are blind to the page, and are only for a canvas with no refs, taken from the @(x,y) the screen shows or from the screenshot.',
     input_schema: {
       type: 'object',
       additionalProperties: false,
       properties: {
-        target: { type: 'string', description: 'web: button "Start", Start Game, or #start-button. Godot: a node path such as /root/Main/UI/StartButton.' },
+        ref: { type: 'string', description: 'playwright_web only, the preferred target: the ref shown on the element, such as e12.' },
+        target: { type: 'string', description: 'web: button "Start" or Start Game. Godot: a node path such as /root/Main/UI/StartButton.' },
         x: { type: 'number' },
         y: { type: 'number' },
         thought_in_character: thought,
@@ -95,7 +102,8 @@ export const PLAY_TOOLS = Object.freeze([
         text: { type: 'string' },
         submit: { type: 'boolean' },
         replace: { type: 'boolean', description: 'Empty the field first, so it holds only this text. playwright_web only; other drivers refuse it.' },
-        target: { type: 'string', description: 'playwright_web only, other drivers refuse it: the field to type into, named like a click target, such as textbox "Email". Without it, the focused field.' },
+        ref: { type: 'string', description: 'playwright_web only, the preferred way to name the field: its ref, such as e12.' },
+        target: { type: 'string', description: 'playwright_web only, other drivers refuse it: the field to type into, named like a click target, such as textbox "Email". Without it or a ref, the focused field.' },
         thought_in_character: thought,
         note
       }
@@ -186,6 +194,7 @@ export function actionFor(toolUse) {
   const input = toolUse.input || {};
   switch (toolUse.name) {
     case 'click':
+      if (typeof input.ref === 'string' && input.ref.trim()) return { kind: 'click', ref: input.ref.trim() };
       return Number.isFinite(Number(input.x)) && Number.isFinite(Number(input.y))
         ? { kind: 'click', target: { x: Number(input.x), y: Number(input.y) } }
         : { kind: 'click', target: String(input.target || '') };
@@ -193,6 +202,7 @@ export function actionFor(toolUse) {
       const action = { kind: 'type', text: String(input.text || ''), submit: Boolean(input.submit) };
       if (input.replace === true) action.replace = true;
       if (typeof input.target === 'string' && input.target.trim()) action.target = input.target;
+      if (typeof input.ref === 'string' && input.ref.trim()) action.ref = input.ref.trim();
       return action;
     }
     case 'key':
@@ -220,9 +230,11 @@ export function actionFor(toolUse) {
  * @param {Object} args.toolUse
  * @param {(note: Object) => void} [args.onNote]
  * @param {(shot: Object) => void} [args.onScreenshot]
+ * @param {boolean} [args.staleRefs] an earlier action in this turn changed the
+ *   screen, so a ref in this call was read off a screen that is gone
  * @returns {Promise<{result: Object, observation: Object|null, spentAction: boolean, changed: boolean, error: string|null}>}
  */
-export async function executeTool({ driver, toolUse, onNote, onScreenshot }) {
+export async function executeTool({ driver, toolUse, onNote, onScreenshot, staleRefs = false, identity = null }) {
   const input = toolUse.input || {};
   if (toolUse.malformed) {
     return {
@@ -260,8 +272,13 @@ export async function executeTool({ driver, toolUse, onNote, onScreenshot }) {
 
   if (toolUse.name === 'snapshot') {
     const observation = await driver.observe();
+    const result = observationResult(toolUse.id, observation);
+    // The run's identity rides on the one tool a persona calls to look
+    // around: the act step's tool allowlist has no room for a tool of its own.
+    const line = identityLine(identity);
+    if (line) result.content.push({ type: 'text', text: line });
     return {
-      result: observationResult(toolUse.id, observation),
+      result,
       observation,
       spentAction: false,
       changed: false,
@@ -280,7 +297,15 @@ export async function executeTool({ driver, toolUse, onNote, onScreenshot }) {
     };
   }
 
-  const outcome = await driver.act(action);
+  // A ref is a playwright_web target. Every other driver is told so by name,
+  // before anything reaches the game.
+  // A ref in a later tool call of a turn whose earlier call already changed
+  // the screen was read off a screen that is gone.
+  const outcome = action.ref && !driver.supportsRefs
+    ? { ok: false, changed: false, error: `ref targets are only available on playwright_web; ${driver.name || 'this driver'} takes a named target or coordinates`, driverError: true }
+    : action.ref && staleRefs
+      ? { ok: false, changed: false, error: STALE_REF_MESSAGE, driverError: true }
+      : await driver.act(action);
   const observation = await driver.observe();
   const spentAction = ACTION_TOOLS.includes(toolUse.name) && outcome.ok;
   const prefix = outcome.ok
@@ -291,7 +316,8 @@ export async function executeTool({ driver, toolUse, onNote, onScreenshot }) {
     observation,
     spentAction,
     changed: Boolean(outcome.changed),
-    error: outcome.ok ? null : String(outcome.error || 'act failed')
+    error: outcome.ok ? null : String(outcome.error || 'act failed'),
+    driverError: !outcome.ok && Boolean(outcome.driverError)
   };
 }
 
@@ -330,6 +356,17 @@ export function observationResult(id, observation, prefix = '', isError = false)
   return { type: 'tool_result', tool_use_id: id, content, ...(isError ? { is_error: true } : {}) };
 }
 
+/**
+ * The run's identity as one line of a snapshot result, or '' with none.
+ *
+ * @param {{email?: string, displayName?: string}|null} identity
+ * @returns {string}
+ */
+export function identityLine(identity) {
+  if (!identity || !identity.email) return '';
+  return `run identity: email ${identity.email}, display name ${identity.displayName}. Use these whenever a form asks for an email or a name.`;
+}
+
 /**
  * @param {string} id
  * @param {string} text

package/src/run/usage.js

@@ -29,6 +29,7 @@ export function createUsageLedger() {
   const byModel = new Map();
   const started = Date.now();
   let calls = 0;
+  let modelMs = 0;
   let serverTotalUsd = null;
   let serverCalls = null;
 
@@ -36,8 +37,10 @@ export function createUsageLedger() {
     /**
      * @param {{model: string|null, usage: Object}} turn
      */
-    record({ model, usage }) {
+    record({ model, usage, ms }) {
       calls += 1;
+      // Time spent waiting on the model, when the caller measured it.
+      if (Number.isFinite(Number(ms)) && Number(ms) > 0) modelMs += Number(ms);
       const key = model || 'unknown';
       const entry = byModel.get(key) || { model: key, input_tokens: 0, cache_read_tokens: 0, output_tokens: 0, usd: 0 };
       // The report schema has no cache-creation field, so creation tokens
@@ -64,6 +67,20 @@ export function createUsageLedger() {
       return calls;
     },
 
+    /**
+     * Wall time since the ledger was made, and time spent waiting on model
+     * calls, in seconds. What `complete` sends as `duration`, so the run card
+     * does not read 0s for a run that played for minutes.
+     *
+     * @returns {{wall_seconds: number, model_seconds: number}}
+     */
+    duration() {
+      return {
+        wall_seconds: Number(((Date.now() - started) / 1000).toFixed(3)),
+        model_seconds: Number((modelMs / 1000).toFixed(3))
+      };
+    },
+
     /**
      * The `usage` object as the report schema wants it.
      *
@@ -87,6 +104,7 @@ export function createUsageLedger() {
       const report = this.toReportUsage();
       return {
         ...report,
+        model_seconds: Number((modelMs / 1000).toFixed(3)),
         local_model_calls: calls,
         server_model_calls: serverCalls,
         usd_source: serverTotalUsd === null ? 'unavailable' : 'server',