ravensight-playtest 0.2.9 → 0.3.0

package/src/run/personaLoop.js

@@ -1,3 +1,4 @@
+import { rewriteMarkdown } from './dropped.js';
 import { selectAction, JEV_CONTROLLER_PROMPT } from './actionSelector.js';
 import { mkdir, writeFile } from 'node:fs/promises';
 import path from 'node:path';
@@ -69,6 +70,8 @@ const ABOVE_MINOR = Object.freeze(['blocker', 'major']);
 export const NETWORK_CAP_NOTE = 'Severity capped at minor by the runner: the only evidence is automatic network notes, and a failed request is a finding above minor only when the persona saw the game break because of it.';
 
 /** A persona note kind as a report finding category. */
+// `waiting` has no category on purpose: a "waiting for X to finish" note is
+// context, and nothing turns it into a finding.
 export const NOTE_CATEGORY = Object.freeze({ bug: 'bug', confusing: 'ux', suggestion: 'ux', good: 'ux', reaction: 'ux' });
 
 /** Stop injecting the "nothing changed" hint after this many identical screens. */
@@ -77,8 +80,16 @@ export const STUCK_HINT_AFTER = 5;
 /** Give up entirely after this many identical screens: the persona is wedged. */
 export const STUCK_LIMIT = 12;
 
-/** Turns allowed after a budget warning, to land the report. */
-export const WRAP_UP_TURNS = 3;
+/**
+ * Act turns allowed after a budget warning: none. The proxy reserves each
+ * call's pessimistic projection (the whole request at the full input rate,
+ * plus the output ceiling) before it sends it, so every act turn after the
+ * warning ate into the 20 percent the report step needs, and in the Texas run
+ * three of them left too little for the report's own reservation: the report
+ * call was refused with 402 and the run ended at cost_cap with no report.
+ * The warned turn's own actions are still played; then the report is written.
+ */
+export const WRAP_UP_TURNS = 0;
 
 /** Token ceiling asked for per act turn. The proxy clamps it to the step's own. */
 export const ACT_MAX_TOKENS = 1024;
@@ -330,6 +341,7 @@ async function playLoop({
   keepObservations,
   heartbeat,
   onProgress = () => {},
+  onBudgetWarning = null,
   log = () => {}
 }) {
   const system = buildSystem({ skill, persona, brief, intentNotes, driverName, target, maxActions });
@@ -361,7 +373,6 @@ async function playLoop({
   let budgetExceeded = false;
   let canceled = false;
   let unchanged = 0;
-  let wrapUpLeft = Infinity;
   const maxTurns = maxActions * 3 + 20;
 
   // Notes waiting for the next observation, which is the screen they are
@@ -405,13 +416,9 @@ async function playLoop({
       quitDetail = 'the job was cancelled';
       break;
     }
-    if (wrapUpLeft <= 0) {
-      quitReason = 'budget_exhausted';
-      break;
-    }
-
     turns += 1;
     let answer;
+    const askedAt = Date.now();
     try {
       answer = await callWithImageRetry({
         client,
@@ -444,30 +451,28 @@ async function playLoop({
       throw error;
     }
 
-    ledger.record({ model: answer.model, usage: answer.usage });
+    ledger.record({ model: answer.model, usage: answer.usage, ms: Date.now() - askedAt });
     // The warn turn is a turn the developer paid for, so its tool calls are
     // executed like any other and the wrap-up notice rides along on that
     // turn's tool_result. Two things this avoids: discarding a paid action,
     // and pushing a second consecutive `user` message, which the Messages API
     // refuses when the previous turn ended in a tool_use.
-    let wrapUpNotice = null;
-    if (answer.budget === 'warn' && !budgetWarned) {
+    const warnedNow = answer.budget === 'warn' && !budgetWarned;
+    if (warnedNow) {
       budgetWarned = true;
-      wrapUpLeft = WRAP_UP_TURNS;
       // Surfaced, not swallowed: this is the developer's money and the one
       // moment where telling them costs nothing and saying nothing costs a
-      // report.
-      log(`budget warning: this run has used at least 80 percent of its model budget. Wrapping up after ${WRAP_UP_TURNS} more turns so the report still gets written.`);
-      wrapUpNotice = `You are nearly out of budget for this session. Take a final screenshot if you need one, then stop calling tools so the report can be written. You have ${WRAP_UP_TURNS} turns left.`;
+      // report. Recorded before anything else can fail, so the journal and
+      // the transcript both say the run was warned.
+      log('budget warning: this run has used at least 80 percent of its model budget. Playing this turn\'s actions, then stopping so the rest of the budget goes to the report.');
       transcript.write({ kind: 'budget_warning', wrap_up_turns: WRAP_UP_TURNS });
+      if (typeof onBudgetWarning === 'function') await onBudgetWarning();
     }
-    if (budgetWarned) wrapUpLeft -= 1;
 
     if (answer.toolUses.length === 0) {
       // No tool call is how a persona stops: there is no quit tool in the
-      // step's allowlist, and adding one would be refused by the proxy. A warn
-      // on such a turn needs no notice either, since the loop ends here.
-      quitReason = budgetWarned && wrapUpNotice ? 'budget_exhausted' : 'persona_quit';
+      // step's allowlist, and adding one would be refused by the proxy.
+      quitReason = warnedNow ? 'budget_exhausted' : 'persona_quit';
       quitDetail = answer.text.slice(0, 400);
       transcript.write({ kind: 'quit', reason: quitReason, text: answer.text });
       break;
@@ -484,15 +489,25 @@ async function playLoop({
       }
       transcript.write({ kind: 'action_selection', source: selection.source || 'jev', status: selection.status, confidence: selection.confidence, selected_tool_id: selection.toolId, reason: selection.reason });
       if (selection.status !== 'selected') { quitReason = 'persona_quit'; quitDetail = 'Ravensight Playtester stopped for review: ' + selection.reason; break; }
-      if (selection.budget === 'warn') { budgetWarned = true; wrapUpLeft = Math.min(wrapUpLeft, WRAP_UP_TURNS); }
+      if (selection.budget === 'warn' && !budgetWarned) {
+        budgetWarned = true;
+        log('budget warning: this run has used at least 80 percent of its model budget. Playing this turn\'s actions, then stopping so the rest of the budget goes to the report.');
+        transcript.write({ kind: 'budget_warning', wrap_up_turns: WRAP_UP_TURNS });
+        if (typeof onBudgetWarning === 'function') await onBudgetWarning();
+      }
     }
     messages.push({ role: 'assistant', content: assistantContent(answer) });
     const results = [];
+    // Every ref in this turn was read off the screen before it. Once one of
+    // its actions changes the screen, a later ref may name something else.
+    let screenChangedThisTurn = false;
     for (const toolUse of answer.toolUses) {
       if (selection && toolUse.id !== selection.toolId) { results.push({ type: 'tool_result', tool_use_id: toolUse.id, content: 'Alternative not selected. No action executed.' }); continue; }
+      const notesBefore = notes.length;
       const executed = await executeTool({
         driver,
         toolUse,
+        staleRefs: screenChangedThisTurn,
         onNote: note => {
           // The note is nested rather than spread: its own `kind`
           // (bug, confusing, good, suggestion, reaction) would otherwise
@@ -508,6 +523,7 @@ async function playLoop({
       });
       results.push(executed.result);
       if (executed.spentAction) actions += 1;
+      if (executed.changed && ACTION_TOOLS.includes(toolUse.name)) screenChangedThisTurn = true;
       transcript.write({
         kind: 'action',
         tool: toolUse.name,
@@ -518,8 +534,18 @@ async function playLoop({
         changed: executed.changed,
         actions
       });
+      if (executed.driverError) {
+        // The driver failed, not the game: a target that matched twice,
+        // locator code, a ref not on screen, a timeout on an unchanged page.
+        // Recorded as its own kind, and the observation and any note written
+        // on this call are marked, so the report step can be told (and the
+        // validator can check) that none of it is evidence of a game bug.
+        for (const note of notes.slice(notesBefore)) note.driver_error = true;
+        transcript.write({ kind: 'driver_error', tool: toolUse.name, error: executed.error, observation: observations.length + (executed.observation ? 1 : 0) });
+      }
       if (executed.observation) {
-        observed(executed.observation);
+        const entry = observed(executed.observation);
+        if (executed.driverError) entry.driver_error = String(executed.error || 'driver error').slice(0, 200);
         latestObservation = executed.observation.text;
         visualObservation = Boolean(executed.observation.screenshotBase64);
         recentActions.push(`${toolUse.name}: ${executed.changed ? 'screen changed' : 'no observed change'}`);
@@ -561,8 +587,6 @@ async function playLoop({
       asides.push(`The game logged ${consoleErrors.length} error(s): ${consoleErrors.map(entry => entry.text).join(' | ').slice(0, 800)}${shot ? ` A screenshot is saved at ${shot.relativePath}.` : ''}`);
     }
 
-    if (wrapUpNotice) asides.push(wrapUpNotice);
-
     const wedged = unchanged >= STUCK_LIMIT;
     if (!wedged && unchanged > 0 && unchanged % STUCK_HINT_AFTER === 0) {
       transcript.write({ kind: 'stuck', unchanged, fatal: false });
@@ -584,6 +608,12 @@ async function playLoop({
     }
 
     if (heartbeat) await heartbeat.beat({ actionsTaken: actions, checkpointStep: turns });
+
+    if (budgetWarned) {
+      quitReason = 'budget_exhausted';
+      quitDetail = 'stopped at the 80 percent budget warning so the report could be written';
+      break;
+    }
   }
 
   if (actions >= maxActions && quitReason === 'budget_exhausted') {
@@ -645,6 +675,8 @@ async function reportPhase({
         notes.length > 0 ? notes.map((note, i) => noteLine(note, i)).join('\n') : '(none)',
         '',
         'Notes marked automatic were recorded by the runner from the console, with no severity: decide one from what the persona saw. Network notes are context, not bugs: a failed request is a finding only when the persona saw a broken screen because of it. A finding whose only evidence is automatic network notes is capped at minor by the runner.',
+        'Driver errors are never evidence of a game bug. An observation marked [driver error: ...] or a note "written on a driver error" means the runner could not carry out the action (a target that matched two elements, locator code, a ref not on screen, a timeout on a page that did not change); the game did nothing wrong there. Never report a blocked loop, a frozen screen or a broken button from them. The runner drops any blocker or major whose only cited observations are driver errors.',
+        'Notes of kind waiting ("waiting for X to finish") are context, never a finding of any severity, and the runner drops a finding cited only by them. Before reporting a loading problem, check the later observations and screenshots: if they show the content loaded, it is not a finding.',
         '',
         'Observations the persona was shown, by index:',
         observationLines(observations),
@@ -674,6 +706,7 @@ async function reportPhase({
   let jsonText = null;
   let repaired = false;
   let continued = false;
+  let droppedByRunner = [];
 
   // One repair at most when the schema or the evidence refs refuse what was
   // written: each turn is a metered question, and the dogfood gate asks for
@@ -682,19 +715,56 @@ async function reportPhase({
   // gets one plain "now write report.json" that does not spend the repair.
   // In 0.2.8 it did, and the report.json that came back had its five
   // screenshot refs under "path" with no turn left to fix them.
+  // A warned run has at most a fifth of its budget left, and the proxy
+  // reserves each call's worst case before sending it. So the report step
+  // never escalates to a dearer tier after a warning, and a 402 on a report
+  // turn gets one retry with the output ceiling cut to what the refusal says
+  // still fits.
+  const escalate = shouldEscalate(persona) && !play.budgetWarned;
+  let maxTokens = REPORT_MAX_TOKENS;
+  let shrunk = false;
+  let lastCandidate = null;
   for (let attempt = 1; ; attempt += 1) {
-    const answer = await callStepWithRetry({
-      client,
-      jobId,
-      runId,
-      step: STEPS.personaReport,
-      system,
-      messages,
-      tools: REPORT_TOOLS,
-      maxTokens: REPORT_MAX_TOKENS,
-      escalate: shouldEscalate(persona)
-    });
-    ledger.record({ model: answer.model, usage: answer.usage });
+    let answer;
+    const askedAt = Date.now();
+    try {
+      answer = await callStepWithRetry({
+        client,
+        jobId,
+        runId,
+        step: STEPS.personaReport,
+        system,
+        messages,
+        tools: REPORT_TOOLS,
+        maxTokens,
+        escalate
+      });
+    } catch (error) {
+      if (!(error instanceof ProxyError && error.isBudget)) throw error;
+      const bodyChars = JSON.stringify({ messages, system, tools: REPORT_TOOLS, max_tokens: maxTokens, stream: true }).length;
+      const smaller = shrunk ? null : reportTokensThatFit(error, maxTokens, bodyChars);
+      if (!smaller && !shrunk && !lastCandidate) {
+        log(hasBudgetFigures(error)
+          ? `even the smallest report (${REPORT_MIN_TOKENS} output tokens) does not fit the remaining model budget once the request itself is paid for, so the report is written from the notes instead.`
+          : 'the report turn was refused for budget with no budget figures in the refusal, so there is nothing to size a retry from; the report is written from the notes instead.');
+      }
+      if (smaller) {
+        shrunk = true;
+        transcript.write({ kind: 'report_budget_retry', attempt, max_tokens: smaller, figures: error.figures || {} });
+        log(`the report turn did not fit the remaining model budget; asking again with an output ceiling of ${smaller} tokens.`);
+        maxTokens = smaller;
+        continue;
+      }
+      // A report already written on an earlier turn is the report, with its
+      // lint, rather than a partial one rebuilt from the notes.
+      if (lastCandidate) {
+        report = lastCandidate;
+        log(`the model budget ran out on a follow-up report turn; keeping the report.json already written (${lint.length} problem(s)).`);
+        break;
+      }
+      throw error;
+    }
+    ledger.record({ model: answer.model, usage: answer.usage, ms: Date.now() - askedAt });
     transcript.write({ kind: 'report_turn', attempt, stop_reason: answer.stopReason, files: answer.toolUses.map(t => t.input && t.input.path) });
 
     const files = new Map();
@@ -737,7 +807,14 @@ async function reportPhase({
     if (normalised && normalised.changed > 0) {
       transcript.write({ kind: 'report_evidence_normalised', attempt, entries: normalised.changed });
     }
-    const candidate = normalised ? capNetworkOnlyFindings(stampReport(normalised.report, facts), { notes, screenshots }) : null;
+    const capped = normalised ? capNetworkOnlyFindings(stampReport(normalised.report, facts), { notes, screenshots }) : null;
+    const runnerDrop = capped ? dropRunnerRejectedFindings(capped, { notes, observations }) : { report: null, dropped: [] };
+    const candidate = runnerDrop.report;
+    for (const entry of runnerDrop.dropped) {
+      transcript.write({ kind: 'finding_dropped', id: entry.finding.id, severity: entry.finding.severity, title: entry.finding.title, reason: entry.reason });
+    }
+    droppedByRunner = runnerDrop.dropped;
+    if (candidate) lastCandidate = candidate;
     const validation = candidate ? validator.validate(candidate) : { ok: false, details: [{ instancePath: '/', message: parseError || 'report.json was not written' }] };
     const evidenceProblems = candidate ? await checkEvidence(candidate, runDirectory) : [];
     // The server lints the same four sections at `complete` and records the
@@ -798,6 +875,10 @@ async function reportPhase({
   if (!markdown) {
     markdown = renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason: 'The report step wrote report.json but no report.md.' });
   }
+  if (droppedByRunner.length > 0) {
+    markdown = rewriteMarkdown(markdown, droppedByRunner.map(entry => entry.finding), report.findings || [], RUNNER_DROPPED_SECTION);
+    log(`dropped ${droppedByRunner.length} finding(s) the runner's evidence rules refuse: ${droppedByRunner.map(entry => `${entry.finding.id} (${entry.reason})`).join(', ')}`);
+  }
   const generatedAt = new Date().toISOString();
   const facts = reportFacts({ persona, driverName, target, play });
   markdown = stampGeneratedAt(markdown, generatedAt, facts);
@@ -808,6 +889,72 @@ async function reportPhase({
   return { report, markdown, lint };
 }
 
+/** The smallest output ceiling worth retrying a report turn with. */
+export const REPORT_MIN_TOKENS = 1536;
+
+/** Whether a 402 carried the three figures a retry is sized from. */
+export function hasBudgetFigures(error) {
+  const figures = (error && error.figures) || {};
+  return ['usd_used', 'usd_max', 'projected_usd'].every(key => Number.isFinite(Number(figures[key])) && figures[key] !== null && figures[key] !== '');
+}
+
+/** Characters per input token, for estimating the input share of a projection. */
+export const CHARS_PER_INPUT_TOKEN = 3.5;
+
+/**
+ * Output price per token over input price per token, assumed when the 402
+ * does not split its projection. Prices are server side only; this ratio is
+ * only used to divide the server's own projected figure into two parts.
+ */
+export const ASSUMED_OUTPUT_PRICE_RATIO = 5;
+
+/**
+ * The output ceiling a refused report turn could fit in, or null when even
+ * the minimum report cannot fit.
+ *
+ * The proxy's projection prices the input and the output ceiling together,
+ * and only the output ceiling can move, so the input's share is taken out of
+ * the room first: from the 402's own split (`projected: { input_cents,
+ * output_cents }`) when the server sends one, otherwise estimated from the
+ * request's size (characters / 3.5 at an input price a fifth of the output
+ * price). What is left of the room is then shared out over the output, with
+ * a tenth of headroom.
+ *
+ * @param {ProxyError} error
+ * @param {number} current the output ceiling that was refused
+ * @param {number} [bodyChars] the refused request's JSON size
+ * @returns {number|null}
+ */
+export function reportTokensThatFit(error, current, bodyChars = 0) {
+  const figures = (error && error.figures) || {};
+  const used = Number(figures.usd_used);
+  const max = Number(figures.usd_max);
+  const projected = Number(figures.projected_usd);
+  if (![used, max, projected].every(Number.isFinite) || projected <= 0 || current <= 0) return null;
+  const room = max - used;
+  if (room <= 0 || room >= projected) return null;
+  const split = figures.projected && typeof figures.projected === 'object' ? figures.projected : null;
+  let inputUsd;
+  let outputUsd;
+  if (split && Number.isFinite(Number(split.input_cents)) && Number.isFinite(Number(split.output_cents))) {
+    inputUsd = Number(split.input_cents) / 100;
+    outputUsd = Number(split.output_cents) / 100;
+  } else {
+    // The request's JSON size counts any base64 screenshot as text, which
+    // overstates the input (an image is billed far below its base64 length
+    // as text). Overstating the input understates what the output can have,
+    // so the estimate errs on the safe side: a retry it allows should fit.
+    const inputTokens = Math.max(0, Number(bodyChars) || 0) / CHARS_PER_INPUT_TOKEN;
+    const perInputToken = projected / (inputTokens + ASSUMED_OUTPUT_PRICE_RATIO * current);
+    inputUsd = perInputToken * inputTokens;
+    outputUsd = projected - inputUsd;
+  }
+  const outputRoom = room - inputUsd;
+  if (outputRoom <= 0 || outputUsd <= 0) return null;
+  const tokens = Math.floor(current * (outputRoom / outputUsd) * 0.9);
+  return tokens >= REPORT_MIN_TOKENS ? tokens : null;
+}
+
 /**
  * `callStepWithRetry`, plus the one retry that is about our own request
  * rather than the server's mood: `too_many_images` means this step allows
@@ -980,6 +1127,7 @@ async function finish(outcome, ledger, runDirectory, transcript, heartbeat) {
     }
   }
   outcome.usage = ledger.toReportUsage();
+  outcome.duration = ledger.duration();
   await writeUsageFile(runDirectory, ledger);
   transcript.write({ kind: 'end', state: outcome.state, quit_reason: outcome.quitReason, actions: outcome.actionsTaken });
   await transcript.close();
@@ -1068,6 +1216,8 @@ export function notesToFindings(notes, screenshots) {
   for (const note of notes) {
     const category = NOTE_CATEGORY[note.kind];
     if (!category || !note.text) continue;
+    // A blocker or major written on a driver error is the runner's failure.
+    if (note.driver_error && ABOVE_MINOR.includes(note.severity)) continue;
     const text = String(note.text);
     const shot = nearestScreenshot(note, screenshots);
     findings.push({
@@ -1145,6 +1295,7 @@ async function writePartialMarkdown({ runDirectory, persona, driverName, target,
 export function noteLine(note, i) {
   const tags = [
     note.automatic ? 'automatic' : null,
+    note.driver_error ? 'written on a driver error' : null,
     Number.isInteger(note.observation) && note.observation > 0 ? `seen at observation ${note.observation}` : null,
     note.screenshot ? `screenshot ${note.screenshot}` : null
   ].filter(Boolean);
@@ -1182,7 +1333,7 @@ export const MAX_OBSERVATIONS_IN_REPORT = 200;
 function observationLines(observations) {
   if (!observations || observations.length === 0) return '(none)';
   const shown = observations.slice(-MAX_OBSERVATIONS_IN_REPORT);
-  const lines = shown.map(entry => `observation ${entry.index}: step ${entry.step}${entry.url ? ` ${entry.url}` : ''}${entry.title ? ` "${entry.title}"` : ''}`);
+  const lines = shown.map(entry => `observation ${entry.index}: step ${entry.step}${entry.url ? ` ${entry.url}` : ''}${entry.title ? ` "${entry.title}"` : ''}${entry.driver_error ? ` [driver error: ${entry.driver_error}]` : ''}`);
   if (shown.length < observations.length) lines.unshift(`(the first ${observations.length - shown.length} are left out)`);
   return lines.join('\n');
 }
@@ -1215,6 +1366,65 @@ export function observationTitle(observation) {
  * @param {{notes: Array<Object>, screenshots: Array<Object>}} context
  * @returns {Object} the report, with capped findings rewritten
  */
+/** How report.md lists what the runner dropped. */
+export const RUNNER_DROPPED_SECTION = Object.freeze({
+  heading: '## Dropped by the runner (driver errors or waiting notes only)',
+  explanation: 'The runner removes a blocker or major whose only cited observations are driver errors (the runner could not carry out an action; the game did nothing wrong), and any finding cited only by waiting notes. They are not sent to the server. They are kept here so you can judge them yourself.',
+  emptyNote: '- Nothing here that the runner kept. See the dropped findings below.'
+});
+
+/**
+ * The findings the runner refuses before anything is uploaded, and the
+ * report without them.
+ *
+ * - A blocker or major whose transcript evidence (observations and notes it
+ *   cites) all points at driver errors. Screenshots do not rescue it: the
+ *   Texas run filed "the season loop is fully blocked" from three strict mode
+ *   failures, with a screenshot of the screen working.
+ * - Any finding whose transcript evidence is only waiting notes.
+ *
+ * A finding with no transcript evidence is left to the other checks.
+ *
+ * @param {Object} report
+ * @param {{notes?: Array, observations?: Array}} context
+ * @returns {{report: Object, dropped: Array<{finding: Object, reason: string}>}}
+ */
+export function dropRunnerRejectedFindings(report, { notes = [], observations = [] } = {}) {
+  if (!report || !Array.isArray(report.findings)) return { report, dropped: [] };
+  const byIndex = new Map(observations.map(entry => [entry.index, entry]));
+  const verdict = finding => {
+    const cited = [];
+    for (const item of Array.isArray(finding.evidence) ? finding.evidence : []) {
+      if (!item || item.type !== 'transcript_step') continue;
+      const ref = String(item.ref || '');
+      const observation = ref.match(/^\s*observation\s*#?\s*(\d+)\s*$/i);
+      const note = ref.match(/^\s*note\s*#?\s*(\d+)\s*$/i);
+      if (observation) {
+        const entry = byIndex.get(Number(observation[1]));
+        cited.push(entry && entry.driver_error ? 'driver' : 'other');
+      } else if (note) {
+        const entry = notes[Number(note[1]) - 1];
+        cited.push(entry && entry.kind === 'waiting' ? 'waiting' : entry && entry.driver_error ? 'driver' : 'other');
+      } else {
+        cited.push('other');
+      }
+    }
+    if (cited.length === 0) return null;
+    if (ABOVE_MINOR.includes(finding.severity) && cited.every(kind => kind === 'driver')) return 'driver errors only';
+    if (cited.every(kind => kind === 'waiting')) return 'waiting notes only';
+    if (ABOVE_MINOR.includes(finding.severity) && cited.every(kind => kind === 'driver' || kind === 'waiting')) return 'driver errors and waiting notes only';
+    return null;
+  };
+  const dropped = [];
+  const findings = [];
+  for (const finding of report.findings) {
+    const reason = finding ? verdict(finding) : null;
+    if (reason) dropped.push({ finding, reason });
+    else findings.push(finding);
+  }
+  return { report: dropped.length > 0 ? { ...report, findings } : report, dropped };
+}
+
 export function capNetworkOnlyFindings(report, { notes = [], screenshots = [] } = {}) {
   if (!report || !Array.isArray(report.findings)) return report;
   const byShot = new Map();

package/src/run/regenerate.js

@@ -47,7 +47,7 @@ export async function readTranscript(runDirectory) {
   try {
     text = await readFile(path.join(runDirectory, RUN_FILES.transcript), 'utf8');
   } catch {
-    return { header: null, notes: [], screenshots: [], observations: 0, actions: 0 };
+    return { header: null, notes: [], screenshots: [], observations: 0, actions: 0, firstT: null, lastT: null };
   }
 
   let header = null;
@@ -55,6 +55,12 @@ export async function readTranscript(runDirectory) {
   const screenshots = [];
   let observations = 0;
   let actions = 0;
+  // Notes written on the tool call the last `action` line closed, so a
+  // `driver_error` after it can mark them as the live run did.
+  let callNotes = [];
+  let openNotes = [];
+  let firstT = null;
+  let lastT = null;
   for (const line of text.split('\n')) {
     if (!line.trim()) continue;
     let entry;
@@ -65,13 +71,27 @@ export async function readTranscript(runDirectory) {
       // it rather than refusing to read the whole transcript.
       continue;
     }
+    if (Number.isFinite(Number(entry.t))) {
+      if (firstT === null) firstT = Number(entry.t);
+      lastT = Number(entry.t);
+    }
     if (entry.kind === 'header') header = entry;
-    else if (entry.kind === 'note' && entry.note) notes.push(entry.note);
+    else if (entry.kind === 'note' && entry.note) {
+      const note = { ...entry.note };
+      notes.push(note);
+      if (entry.tool) openNotes.push(note);
+    } else if (entry.kind === 'driver_error') {
+      for (const note of callNotes) note.driver_error = true;
+    }
     else if (entry.kind === 'screenshot' && entry.path) screenshots.push({ relativePath: entry.path, reason: entry.reason || '' });
     else if (entry.kind === 'observation') observations += 1;
-    else if (entry.kind === 'action' && Number.isFinite(Number(entry.actions))) actions = Math.max(actions, Number(entry.actions));
+    else if (entry.kind === 'action') {
+      callNotes = openNotes;
+      openNotes = [];
+      if (Number.isFinite(Number(entry.actions))) actions = Math.max(actions, Number(entry.actions));
+    }
   }
-  return { header, notes, screenshots, observations, actions };
+  return { header, notes, screenshots, observations, actions, firstT, lastT };
 }
 
 /**

package/src/run/resume.js

@@ -244,7 +244,13 @@ async function finalizeFromTranscript({ deps, api, gameId, jobId, repoRoot, row,
         actions_taken: report.actions_taken,
         quit_reason: 'error',
         report,
-        report_md: markdown
+        report_md: markdown,
+        // The transcript's own span: its first and last timestamps. No model
+        // time is known for a run rebuilt after the process died.
+        duration: {
+          wall_seconds: transcript.firstT !== null && transcript.lastT !== null ? Number(((transcript.lastT - transcript.firstT) / 1000).toFixed(3)) : null,
+          model_seconds: null
+        }
       }
     });
     if (answer && answer.dropped_findings > 0) {

package/src/run/tools.js

@@ -1,5 +1,7 @@
 import { UNCHANGED_OBSERVATION_TEXT } from './context.js';
-import { OBSERVATION_MAX_CHARS, truncateObservation } from './drivers/driver.js';
+import { OBSERVATION_MAX_CHARS, STALE_REF_MESSAGE, truncateObservation } from './drivers/driver.js';
+
+export { STALE_REF_MESSAGE };
 
 /**
  * The client tool set for `persona_playtest.act`.
@@ -23,8 +25,12 @@ import { OBSERVATION_MAX_CHARS, truncateObservation } from './drivers/driver.js'
  * `snapshot`, `screenshot` and `wait` do not.
  */
 
-/** Note kinds, from spec 03's `log_note`. */
-export const NOTE_KINDS = Object.freeze(['bug', 'confusing', 'good', 'suggestion', 'reaction']);
+/**
+ * Note kinds, from spec 03's `log_note`, plus `waiting`: the persona is
+ * waiting for something to finish (a load, an animation, a turn). A waiting
+ * note is context for the report and never becomes a finding.
+ */
+export const NOTE_KINDS = Object.freeze(['bug', 'confusing', 'good', 'suggestion', 'reaction', 'waiting']);
 
 /** Severities, from the report schema's finding severity enum. */
 export const SEVERITIES = Object.freeze(['blocker', 'major', 'minor', 'cosmetic', 'praise']);
@@ -52,7 +58,7 @@ const note = {
   additionalProperties: false,
   required: ['kind', 'text'],
   properties: {
-    kind: { type: 'string', enum: [...NOTE_KINDS] },
+    kind: { type: 'string', enum: [...NOTE_KINDS], description: 'waiting means you are waiting for something to finish; it is recorded as context and never reported as a finding.' },
     text: { type: 'string' },
     severity: { type: 'string', enum: [...SEVERITIES] }
   }
@@ -71,12 +77,13 @@ export const PLAY_TOOLS = Object.freeze([
   },
   {
     name: 'click',
-    description: 'Click one thing. Spends an action. Either name the target or give x and y pixel coordinates. What a named target means depends on the driver: on playwright_web it is an accessibility node such as button "Start", a visible text fragment, or a CSS selector; on godot_driver it is a node path from the scene tree you were shown.',
+    description: 'Click one thing. Spends an action. On playwright_web, use the ref the screen shows next to the element, such as e12: it names exactly one element, even when two share a name. Otherwise name the target: on playwright_web an accessibility node such as button "Start" or a visible text fragment; on godot_driver a node path from the scene tree you were shown. Targets are refs or visible text, never locator code such as getByRole(...) or page.locator(...). Avoid x and y coordinates on playwright_web: they are blind to the page, and are only for a canvas with no refs, taken from the @(x,y) the screen shows or from the screenshot.',
     input_schema: {
       type: 'object',
       additionalProperties: false,
       properties: {
-        target: { type: 'string', description: 'web: button "Start", Start Game, or #start-button. Godot: a node path such as /root/Main/UI/StartButton.' },
+        ref: { type: 'string', description: 'playwright_web only, the preferred target: the ref shown on the element, such as e12.' },
+        target: { type: 'string', description: 'web: button "Start" or Start Game. Godot: a node path such as /root/Main/UI/StartButton.' },
         x: { type: 'number' },
         y: { type: 'number' },
         thought_in_character: thought,
@@ -95,7 +102,8 @@ export const PLAY_TOOLS = Object.freeze([
         text: { type: 'string' },
         submit: { type: 'boolean' },
         replace: { type: 'boolean', description: 'Empty the field first, so it holds only this text. playwright_web only; other drivers refuse it.' },
-        target: { type: 'string', description: 'playwright_web only, other drivers refuse it: the field to type into, named like a click target, such as textbox "Email". Without it, the focused field.' },
+        ref: { type: 'string', description: 'playwright_web only, the preferred way to name the field: its ref, such as e12.' },
+        target: { type: 'string', description: 'playwright_web only, other drivers refuse it: the field to type into, named like a click target, such as textbox "Email". Without it or a ref, the focused field.' },
         thought_in_character: thought,
         note
       }
@@ -186,6 +194,7 @@ export function actionFor(toolUse) {
   const input = toolUse.input || {};
   switch (toolUse.name) {
     case 'click':
+      if (typeof input.ref === 'string' && input.ref.trim()) return { kind: 'click', ref: input.ref.trim() };
       return Number.isFinite(Number(input.x)) && Number.isFinite(Number(input.y))
         ? { kind: 'click', target: { x: Number(input.x), y: Number(input.y) } }
         : { kind: 'click', target: String(input.target || '') };
@@ -193,6 +202,7 @@ export function actionFor(toolUse) {
       const action = { kind: 'type', text: String(input.text || ''), submit: Boolean(input.submit) };
       if (input.replace === true) action.replace = true;
       if (typeof input.target === 'string' && input.target.trim()) action.target = input.target;
+      if (typeof input.ref === 'string' && input.ref.trim()) action.ref = input.ref.trim();
       return action;
     }
     case 'key':
@@ -220,9 +230,11 @@ export function actionFor(toolUse) {
  * @param {Object} args.toolUse
  * @param {(note: Object) => void} [args.onNote]
  * @param {(shot: Object) => void} [args.onScreenshot]
+ * @param {boolean} [args.staleRefs] an earlier action in this turn changed the
+ *   screen, so a ref in this call was read off a screen that is gone
  * @returns {Promise<{result: Object, observation: Object|null, spentAction: boolean, changed: boolean, error: string|null}>}
  */
-export async function executeTool({ driver, toolUse, onNote, onScreenshot }) {
+export async function executeTool({ driver, toolUse, onNote, onScreenshot, staleRefs = false }) {
   const input = toolUse.input || {};
   if (toolUse.malformed) {
     return {
@@ -280,7 +292,15 @@ export async function executeTool({ driver, toolUse, onNote, onScreenshot }) {
     };
   }
 
-  const outcome = await driver.act(action);
+  // A ref is a playwright_web target. Every other driver is told so by name,
+  // before anything reaches the game.
+  // A ref in a later tool call of a turn whose earlier call already changed
+  // the screen was read off a screen that is gone.
+  const outcome = action.ref && !driver.supportsRefs
+    ? { ok: false, changed: false, error: `ref targets are only available on playwright_web; ${driver.name || 'this driver'} takes a named target or coordinates`, driverError: true }
+    : action.ref && staleRefs
+      ? { ok: false, changed: false, error: STALE_REF_MESSAGE, driverError: true }
+      : await driver.act(action);
   const observation = await driver.observe();
   const spentAction = ACTION_TOOLS.includes(toolUse.name) && outcome.ok;
   const prefix = outcome.ok
@@ -291,7 +311,8 @@ export async function executeTool({ driver, toolUse, onNote, onScreenshot }) {
     observation,
     spentAction,
     changed: Boolean(outcome.changed),
-    error: outcome.ok ? null : String(outcome.error || 'act failed')
+    error: outcome.ok ? null : String(outcome.error || 'act failed'),
+    driverError: !outcome.ok && Boolean(outcome.driverError)
   };
 }