ravensight-playtest 0.7.0 → 0.7.2

package/src/run/gameRefusal.js

@@ -0,0 +1,57 @@
+/**
+ * Which failed actions are the game's state and which are the runner's.
+ *
+ * A web page that shows a control as disabled, hidden or not editable has
+ * refused the action itself: the click failing is what a player would see.
+ * In a real Lab Ledger run the "Run repeat panel" button stayed disabled
+ * after the page said it was ready, and the blocker the persona filed was
+ * dropped as "driver errors only" because Playwright's click timed out on
+ * a button it had found and that said `disabled`. That is evidence.
+ *
+ * A failure where the runner itself broke stays a driver error: a target
+ * that never resolved (a selector or ref problem), a browser or page that
+ * closed or crashed under the runner.
+ */
+
+/** Playwright's actionability waits that name the game's state. */
+const NOT_ACTIONABLE = /element is not (enabled|editable|visible)/i;
+
+/** The element a locator resolved to, from Playwright's call log. */
+const RESOLVED = /locator resolved to (<[^\n]*)/i;
+
+/** A resolved element whose own markup says it is disabled. */
+const RESOLVED_DISABLED = /locator resolved to <[^>]*\s(?:disabled(?:[\s=>/]|$)|aria-disabled="true")/i;
+
+/** The runner's browser, context or page went away under it. */
+const RUNNER_GONE = /Target (?:page, context or browser has been closed|crashed)|browser has (?:been )?disconnected|Browser closed|Page crashed/i;
+
+const STATE_WORDS = { enabled: 'disabled', editable: 'not editable', visible: 'hidden' };
+
+/**
+ * The game's refusal named in a failed action's error, or null when the
+ * failure is not one.
+ *
+ * @param {string} text the driver's error text (a call log may be cut short)
+ * @returns {{state: string, element: string|null}|null}
+ */
+export function gameRefusal(text) {
+  const value = String(text || '');
+  if (!value || RUNNER_GONE.test(value)) return null;
+  const resolved = value.match(RESOLVED);
+  if (!resolved) return null;
+  const element = resolved[1].trim().slice(0, 160);
+  const waited = value.match(NOT_ACTIONABLE);
+  if (waited) return { state: STATE_WORDS[waited[1].toLowerCase()], element };
+  if (RESOLVED_DISABLED.test(value)) return { state: 'disabled', element };
+  return null;
+}
+
+/**
+ * True when the error says the runner's browser or page went away.
+ *
+ * @param {string} text
+ * @returns {boolean}
+ */
+export function runnerGone(text) {
+  return RUNNER_GONE.test(String(text || ''));
+}

package/src/run/personaLoop.js

@@ -6,12 +6,14 @@ import { prepareMessages, MAX_IMAGES_IN_CONTEXT, KEEP_OBSERVATIONS, shrinkImages
 import { withRollingCacheBreakpoint } from './fixLoop.js';
 import { callStepWithRetry, ProxyError, STEPS } from './model.js';
 import { RUN_FILES } from './paths.js';
-import { appendNormalisedMarkdown, checkEvidence, createReportValidator, lintMarkdownSections, normaliseEnums, normaliseEvidence, repairInstruction, REQUIRED_MARKDOWN_SECTIONS, stampReport, writeReportFiles } from './report.js';
+import { reserveDecision } from './budgetReserve.js';
+import { appendNormalisedMarkdown, checkEvidence, createReportValidator, lintMarkdownSections, normaliseEnums, normaliseEvidence, repairInstruction, REQUIRED_MARKDOWN_SECTIONS, stampReport, stripTrailingToolMarkup, writeReportFiles } from './report.js';
 import { assertNoSecrets, scanForSecrets } from './secretScan.js';
 import { ACTION_TOOLS, PLAY_TOOLS, SWIPE_TOOL, REPORT_TOOLS, executeTool } from './tools.js';
 import { createTranscript } from './transcript.js';
 import { createUsageLedger, writeUsageFile } from './usage.js';
 import { fillPlaceholders, fillReportPlaceholders } from './placeholders.js';
+import { gameRefusal } from './gameRefusal.js';
 import { BUILT_IN_NOISE, expectedNoiseForPrompt, isExpectedNoise } from './noise.js';
 import { MIN_PASSWORD_CHARS } from '../briefLocal.js';
 
@@ -102,6 +104,31 @@ export const ACT_MAX_TOKENS = 1024;
 /** Token ceiling asked for when writing the report. */
 export const REPORT_MAX_TOKENS = 4096;
 
+/** The most characters of a persona's closing message kept as its quit detail. */
+export const MAX_QUIT_DETAIL_CHARS = 160;
+
+/**
+ * A persona's closing message as a one-line quit detail: its first sentence,
+ * without markdown, cut to MAX_QUIT_DETAIL_CHARS. A message that is already
+ * report text says only that the persona stopped.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+export function personaQuitDetail(text) {
+  const plain = 'the persona stopped playing';
+  const first = String(text || '').split(/\n/).map(line => line.replace(/[*_`#>]+/g, '').trim()).find(Boolean) || '';
+  if (!first || /report\.(md|json)|write_file|\b(?:writ(?:e|ing)|start(?:ing)? on) (?:the |my )?(?:final )?report\b/i.test(first)) return plain;
+  const sentence = (first.match(/^.*?[.!?](?=\s|$)/) || [first])[0];
+  if (sentence.length <= MAX_QUIT_DETAIL_CHARS) return sentence;
+  const cut = sentence.slice(0, MAX_QUIT_DETAIL_CHARS - 3);
+  const space = cut.lastIndexOf(' ');
+  return `${space > 40 ? cut.slice(0, space) : cut}...`;
+}
+
+/** The stop detail when play ends to keep the report's budget. */
+export const RESERVE_STOP_DETAIL = 'stopped early so the rest of the model budget pays for the report';
+
 /**
  * @typedef {Object} PersonaRunOutcome
  * @property {string} state one of succeeded, failed, canceled, budget_exceeded
@@ -373,6 +400,11 @@ async function playLoop({
   heartbeat,
   onProgress = () => {},
   onBudgetWarning = null,
+  // The run's model budget in USD, from register (`limits.model_usd`). With it
+  // the loop stops early enough that the report still fits; without it (an
+  // older server) only the server's 80 percent warning does.
+  modelBudgetUsd = null,
+  reportSkill = null,
   log = () => {}
 }) {
   const system = buildSystem({ skill, persona, brief, localBrief, intentNotes, driverName, target, maxActions, runId });
@@ -406,9 +438,39 @@ async function playLoop({
   let budgetWarned = false;
   let budgetExceeded = false;
   let reportBudgetAvailable = false;
+  let reserveStopped = false;
   let canceled = false;
   let unchanged = 0;
   const maxTurns = maxActions * 3 + 20;
+  // What the last few act turns really cost, in weighted units.
+  const recentActUnits = [];
+
+  /**
+   * Whether the next act turn would leave too little for the report. Sized
+   * from the act request about to be sent and the report request as it would
+   * be built now; see ./budgetReserve.js.
+   */
+  const reserveCheck = () => {
+    const spend = modelBudgetUsd ? ledger.spendEstimate() : null;
+    if (!spend) return { stop: false, known: false, figures: {} };
+    const prepared = withRollingCacheBreakpoint(prepareMessages(messages, { maxImages: imageBudget, keepObservations }));
+    const actChars = JSON.stringify({ messages: prepared, max_tokens: ACT_MAX_TOKENS, stream: true, system, tools: playTools }).length;
+    const reportRequest = buildReportRequest({
+      skill: reportSkill || skill, persona, brief, localBrief, intentNotes, driverName, target, runId, jobId,
+      play: { actionsTaken: actions, quitReason: 'budget_exhausted', quitDetail: RESERVE_STOP_DETAIL, actionLog: actionLogFrom(actionRecords) },
+      notes, observations, screenshots
+    });
+    const reportChars = JSON.stringify({ messages: reportRequest.messages, max_tokens: REPORT_MAX_TOKENS, stream: true, system: reportRequest.system, tools: REPORT_TOOLS }).length;
+    return reserveDecision({
+      usdMax: modelBudgetUsd,
+      spend,
+      actChars,
+      actMaxTokens: ACT_MAX_TOKENS,
+      actCostUnits: recentActUnits.length > 0 ? Math.max(...recentActUnits) : 0,
+      reportChars,
+      reportMaxTokens: REPORT_MAX_TOKENS
+    });
+  };
 
   // Notes waiting for the next observation, which is the screen they are
   // checked against: a failure the next screen does not show is not a
@@ -454,6 +516,17 @@ async function playLoop({
       quitDetail = 'the job was cancelled';
       break;
     }
+    if (turns > 0) {
+      const reserve = reserveCheck();
+      if (reserve.stop) {
+        reserveStopped = true;
+        quitReason = 'budget_exhausted';
+        quitDetail = RESERVE_STOP_DETAIL;
+        transcript.write({ kind: 'budget_reserve_stop', ...reserve.figures });
+        log(`stopping play after ${actions} action(s) so the rest of the model budget pays for the report (spent about $${reserve.figures.usd_spent} of $${reserve.figures.usd_max}; the report needs about $${reserve.figures.report_reserve_usd}).`);
+        break;
+      }
+    }
     turns += 1;
     let answer;
     const askedAt = Date.now();
@@ -491,6 +564,8 @@ async function playLoop({
     }
 
     ledger.record({ model: answer.model, usage: answer.usage, ms: Date.now() - askedAt });
+    recentActUnits.push(ledger.lastCallUnits || 0);
+    if (recentActUnits.length > 3) recentActUnits.shift();
     // The warn turn is a turn the developer paid for, so its tool calls are
     // executed like any other and the wrap-up notice rides along on that
     // turn's tool_result. Two things this avoids: discarding a paid action,
@@ -513,7 +588,12 @@ async function playLoop({
       // No tool call is how a persona stops: there is no quit tool in the
       // step's allowlist, and adding one would be refused by the proxy.
       quitReason = warnedNow ? 'budget_exhausted' : 'persona_quit';
-      quitDetail = answer.text.slice(0, 400);
+      // The whole closing message is in the transcript. The detail is one
+      // short line: it becomes report.json's quit_detail, and a closing turn
+      // is often the persona starting on its report ("Now writing the final
+      // report. **report.md:** ..."), which in 0.7.1 ended up as a
+      // succeeded run's reason on the dashboard.
+      quitDetail = personaQuitDetail(answer.text);
       transcript.write({ kind: 'quit', reason: quitReason, text: answer.text });
       break;
     }
@@ -695,6 +775,7 @@ async function playLoop({
     budgetWarned,
     budgetExceeded,
     reportBudgetAvailable,
+    reserveStopped,
     canceled,
     imageBudget,
     finalScreenshot: final ? final.relativePath : null,
@@ -733,55 +814,7 @@ async function reportPhase({
   // The report step never sees the test account: it has nothing to sign in
   // to, and the password crosses the model proxy during play only.
   const secrets = secretsOf(localBrief);
-  const reportLocal = localBrief && Array.isArray(localBrief.expected_noise) ? { expected_noise: localBrief.expected_noise } : {};
-  const system = buildSystem({ skill, persona, brief, localBrief: reportLocal, intentNotes, driverName, target, maxActions: play.actionsTaken, runId });
-  const messages = [{
-    role: 'user',
-    content: [{
-      type: 'text',
-      text: [
-        `The session is over. Write report.md and report.json for run ${runId} of job ${jobId}.`,
-        `persona: ${persona.slug}   driver: ${driverName}   target: ${target}`,
-        `actions taken: ${play.actionsTaken}   stop reason: ${play.quitReason}${play.quitDetail ? ` (${play.quitDetail})` : ''}`,
-        '',
-        'Notes you logged, in order:',
-        notes.length > 0 ? notes.map((note, i) => noteLine(note, i)).join('\n') : '(none)',
-        '',
-        'Notes marked automatic were recorded by the runner from the console, with no severity: decide one from what the persona saw. Network notes are context, not bugs: a failed request is a finding only when the persona saw a broken screen because of it. A finding whose only evidence is automatic network notes is capped at minor by the runner.',
-        'Driver errors are never evidence of a game bug. An observation marked [driver error: ...] or a note "written on a driver error" means the runner could not carry out the action (a target that matched two elements, locator code, a ref not on screen, a timeout on a page that did not change); the game did nothing wrong there. Never report a blocked loop, a frozen screen or a broken button from them. The runner drops any blocker or major whose only cited observations are driver errors.',
-        `Notes marked expected noise match the brief's expected_noise or the built in rule (${BUILT_IN_NOISE}); they are never a finding of any severity, and the runner drops a finding whose transcript evidence is only expected noise.`,
-        'Notes of kind waiting ("waiting for X to finish") are context, never a finding of any severity, and the runner drops a finding cited only by them. Before reporting a loading problem, check the later observations and screenshots: if they show the content loaded, it is not a finding.',
-        '',
-        'Observations the persona was shown, by index:',
-        observationLines(observations),
-        '',
-        'Actions you took (every action that went through, numbered by action; "screen changed" means the screen after it differed from the screen before):',
-        actionLogLines(play.actionLog),
-        '',
-        'A claim in report.md or report.json that something was never done, never worked, did nothing or could not be done must agree with this list. If the list shows you clicked a control and the screen changed, do not write that you never clicked it or that it did nothing; describe what the screen showed after it instead. The runner downgrades a finding that says otherwise.',
-        '',
-        'Network notes (failed requests the runner recorded, with the URL and the status; list them under a "Network notes" heading in report.md when there are any):',
-        networkNoteLines(notes),
-        '',
-        'Each note carries "after": the URL and title of the observation that followed it. A blocker or major finding must cite the observation index where the failure is visible, as evidence {"type": "transcript_step", "ref": "observation N"}, and must not be contradicted by its note\'s "after" state or by a later persona note: if the next screen or a later note shows the game working, it is not a blocker or a major. Cite a note as {"type": "transcript_step", "ref": "note N"}.',
-        'A blocker or major finding must also cite the screenshot taken at or after that observation (the list below says which observation each one followed), and that screenshot must not contradict it: if it shows the thing the finding says is missing or broken, such as the field or the button, it is not a blocker or a major. A text observation can be cut short on a long page; the screenshot is what the player saw. After you write the report, the runner shows each blocker or major (at most three) its cited screenshot and downgrades one step any finding the image contradicts.',
-        '',
-        'Every evidence entry is an object with exactly two keys, "type" and "ref", and nothing else. A screenshot is {"type": "screenshot", "ref": "screenshots/04-example.png"}, never "path" or "file".',
-        '',
-        'Never write calendar dates or times in the prose of report.md or report.json. The runner stamps when the report was generated, so leave the template\'s Date out rather than writing a placeholder for it.',
-        'Replace every <...> slot in the template with real content. Never copy a <...> placeholder into the report.',
-        '',
-        'Screenshots available as evidence (use these exact paths in evidence refs):',
-        screenshots.length > 0 ? screenshots.map(screenshotLine).join('\n') : '(none)',
-        '',
-        `report.md must contain these section headings, spelled exactly: ${REQUIRED_MARKDOWN_SECTIONS.map(section => `"${section}"`).join(', ')}.`,
-        'Write report.md first, then report.json. Leave usage as zeros; the runner fills it in.'
-      ].join('\n')
-    }]
-  }];
-  // The prompt is built from notes, observations and the action log; none
-  // of them may carry the test account password into the report step.
-  messages[0].content[0].text = scrubSecrets(messages[0].content[0].text, secrets);
+  const { system, messages } = buildReportRequest({ skill, persona, brief, localBrief, intentNotes, driverName, target, runId, jobId, play, notes, observations, screenshots });
 
   let report = null;
   let markdown = null;
@@ -808,7 +841,7 @@ async function reportPhase({
   // never escalates to a dearer tier after a warning, and a 402 on a report
   // turn gets one retry with the output ceiling cut to what the refusal says
   // still fits.
-  const escalate = shouldEscalate(persona) && !play.budgetWarned && !play.budgetExceeded;
+  const escalate = shouldEscalate(persona) && !play.budgetWarned && !play.budgetExceeded && !play.reserveStopped;
   let maxTokens = REPORT_MAX_TOKENS;
   let shrunk = false;
   let lastCandidate = null;
@@ -868,7 +901,9 @@ async function reportPhase({
       if (toolUse.name !== 'write_file' || toolUse.malformed) continue;
       const name = String(toolUse.input.path || '');
       if (name !== RUN_FILES.reportMd && name !== RUN_FILES.reportJson) continue;
-      files.set(name, String(toolUse.input.content || ''));
+      // A model sometimes closes its tool call inside the file's content;
+      // the stray tags come off before the file is validated or written.
+      files.set(name, stripTrailingToolMarkup(String(toolUse.input.content || '')));
     }
     if (files.has(RUN_FILES.reportMd)) markdown = files.get(RUN_FILES.reportMd);
     if (files.has(RUN_FILES.reportJson)) jsonText = files.get(RUN_FILES.reportJson);
@@ -1168,6 +1203,60 @@ export async function checkClaimsAgainstScreenshots({ client, jobId, runId, repo
   return { report: { ...report, findings }, downgraded };
 }
 
+/**
+ * The report step's request: its system prompt and its one user message. Built
+ * the same way for the call and for sizing it during play (./budgetReserve.js),
+ * so the budget kept back for the report is the report's own size.
+ *
+ * @returns {{system: Array, messages: Array}}
+ */
+export function buildReportRequest({ skill, persona, brief, localBrief = {}, intentNotes, driverName, target, runId, jobId, play, notes, observations = [], screenshots }) {
+  const secrets = secretsOf(localBrief);
+  const reportLocal = localBrief && Array.isArray(localBrief.expected_noise) ? { expected_noise: localBrief.expected_noise } : {};
+  const system = buildSystem({ skill, persona, brief, localBrief: reportLocal, intentNotes, driverName, target, maxActions: play.actionsTaken, runId });
+  const text = [
+    `The session is over. Write report.md and report.json for run ${runId} of job ${jobId}.`,
+    `persona: ${persona.slug}   driver: ${driverName}   target: ${target}`,
+    `actions taken: ${play.actionsTaken}   stop reason: ${play.quitReason}${play.quitDetail ? ` (${play.quitDetail})` : ''}`,
+    '',
+    'Notes you logged, in order:',
+    notes.length > 0 ? notes.map((note, i) => noteLine(note, i)).join('\n') : '(none)',
+    '',
+    'Notes marked automatic were recorded by the runner from the console, with no severity: decide one from what the persona saw. Network notes are context, not bugs: a failed request is a finding only when the persona saw a broken screen because of it. A finding whose only evidence is automatic network notes is capped at minor by the runner.',
+    'Driver errors are never evidence of a game bug. An observation marked [driver error: ...] or a note "written on a driver error" means the runner could not carry out the action (a target that matched two elements, locator code, a ref not on screen, a timeout on a page that did not change); the game did nothing wrong there. Never report a blocked loop, a frozen screen or a broken button from them. The runner drops any blocker or major whose only cited observations are driver errors.',
+    `Notes marked expected noise match the brief's expected_noise or the built in rule (${BUILT_IN_NOISE}); they are never a finding of any severity, and the runner drops a finding whose transcript evidence is only expected noise.`,
+    'Notes of kind waiting ("waiting for X to finish") are context, never a finding of any severity, and the runner drops a finding cited only by them. Before reporting a loading problem, check the later observations and screenshots: if they show the content loaded, it is not a finding.',
+    '',
+    'Observations the persona was shown, by index:',
+    observationLines(observations),
+    '',
+    'Actions you took (every action that went through, numbered by action; "screen changed" means the screen after it differed from the screen before):',
+    actionLogLines(play.actionLog),
+    '',
+    'A claim in report.md or report.json that something was never done, never worked, did nothing or could not be done must agree with this list. If the list shows you clicked a control and the screen changed, do not write that you never clicked it or that it did nothing; describe what the screen showed after it instead. The runner downgrades a finding that says otherwise.',
+    '',
+    'Network notes (failed requests the runner recorded, with the URL and the status; list them under a "Network notes" heading in report.md when there are any):',
+    networkNoteLines(notes),
+    '',
+    'Each note carries "after": the URL and title of the observation that followed it. A blocker or major finding must cite the observation index where the failure is visible, as evidence {"type": "transcript_step", "ref": "observation N"}, and must not be contradicted by its note\'s "after" state or by a later persona note: if the next screen or a later note shows the game working, it is not a blocker or a major. Cite a note as {"type": "transcript_step", "ref": "note N"}.',
+    'A blocker or major finding must also cite the screenshot taken at or after that observation (the list below says which observation each one followed), and that screenshot must not contradict it: if it shows the thing the finding says is missing or broken, such as the field or the button, it is not a blocker or a major. A text observation can be cut short on a long page; the screenshot is what the player saw. After you write the report, the runner shows each blocker or major (at most three) its cited screenshot and downgrades one step any finding the image contradicts.',
+    '',
+    'Every evidence entry is an object with exactly two keys, "type" and "ref", and nothing else. A screenshot is {"type": "screenshot", "ref": "screenshots/04-example.png"}, never "path" or "file".',
+    '',
+    'Never write calendar dates or times in the prose of report.md or report.json. The runner stamps when the report was generated, so leave the template\'s Date out rather than writing a placeholder for it.',
+    'Replace every <...> slot in the template with real content. Never copy a <...> placeholder into the report.',
+    '',
+    'Screenshots available as evidence (use these exact paths in evidence refs):',
+    screenshots.length > 0 ? screenshots.map(screenshotLine).join('\n') : '(none)',
+    '',
+    `report.md must contain these section headings, spelled exactly: ${REQUIRED_MARKDOWN_SECTIONS.map(section => `"${section}"`).join(', ')}.`,
+    'Write report.md first, then report.json. Leave usage as zeros; the runner fills it in.'
+  ].join('\n');
+  // The prompt is built from notes, observations and the action log; none
+  // of them may carry the test account password into the report step.
+  return { system, messages: [{ role: 'user', content: [{ type: 'text', text: scrubSecrets(text, secrets) }] }] };
+}
+
 /** The smallest output ceiling worth retrying a report turn with. */
 export const REPORT_MIN_TOKENS = 1536;
 
@@ -1553,7 +1642,7 @@ async function finish(outcome, ledger, runDirectory, transcript, heartbeat) {
  * screenshot list and says plainly that it is partial, rather than
  * inventing a persona verdict nobody gave.
  */
-export function renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, actions = [], play, reason }) {
+export function renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, actions = [], actionRecords = [], play, reason, usage = null }) {
   const good = notes.filter(note => note.kind === 'good');
   const bad = notes.filter(note => note.kind === 'bug' || note.kind === 'confusing');
   const network = notes.filter(note => note.automatic && note.url);
@@ -1565,13 +1654,17 @@ export function renderFallbackMarkdown({ persona, driverName, target, notes, scr
   // report. A partial report is still a report somebody reads, and a lint
   // finding on it would be noise about a run that already went wrong.
   return [
-    `# Playtest: ${persona.name || persona.slug} (partial)`,
+    `# Playtest: ${persona.name || persona.slug} (partial, written from the run log)`,
     `**Driver:** ${driverName} - **Target:** ${target}`,
     `**Session length:** ${play.actionsTaken} actions / ${play.quitReason}`,
     '',
     '## Why this report is partial',
     reason,
     '',
+    inCharacterFromLog({ persona, notes, actionRecords, play }),
+    '',
+    dataPointsFromLog({ notes, screenshots, actions, play, usage }),
+    '',
     ...(actions.length > 0 ? ['## What the persona did', actionLogLines(actions), ''] : []),
     '## What worked',
     list(good),
@@ -1592,6 +1685,103 @@ export function renderFallbackMarkdown({ persona, driverName, target, notes, scr
   ].join('\n');
 }
 
+/** The most lines of the persona's own words a log-written section quotes. */
+export const MAX_LOG_VOICE_LINES = 16;
+
+/**
+ * The persona's mood, read off its own notes: what a report written from the
+ * log says it felt, and the sentiment report.json carries. Only the notes the
+ * persona wrote count: an automatic console note is not a feeling, and a note
+ * written on a driver error is the runner's failure.
+ *
+ * @param {Array<Object>} notes
+ * @returns {{counts: Object, sentiment: {fun: number, clarity: number, frustration: number}, wouldContinue: boolean, words: string, known: boolean}}
+ */
+export function moodFromNotes(notes = []) {
+  const own = (notes || []).filter(note => note && !note.automatic && !note.driver_error && !note.expected_noise);
+  const counts = { good: 0, bug: 0, confusing: 0, suggestion: 0, reaction: 0, waiting: 0 };
+  for (const note of own) if (note.kind in counts) counts[note.kind] += 1;
+  const clamp = value => Math.min(5, Math.max(1, Math.round(value)));
+  const troubles = counts.bug + counts.confusing;
+  const known = counts.good + troubles + counts.reaction + counts.suggestion > 0;
+  if (!known) {
+    return { counts, sentiment: { fun: 3, clarity: 3, frustration: 3 }, wouldContinue: false, words: 'unknown: the persona logged no notes about how it was going', known };
+  }
+  const sentiment = {
+    fun: clamp(3 + counts.good - counts.bug - counts.confusing / 2),
+    clarity: clamp(4 - counts.confusing - counts.bug / 2),
+    frustration: clamp(1 + counts.bug + counts.confusing)
+  };
+  const words = counts.good > troubles
+    ? 'mostly positive'
+    : troubles > counts.good
+      ? (counts.good > 0 ? 'mixed, leaning frustrated' : 'mostly frustrated or confused')
+      : 'mixed';
+  return { counts, sentiment, wouldContinue: counts.good > troubles, words, known };
+}
+
+/**
+ * The in-character section of a report written from the log: the persona's
+ * own words while it played (the thought before each action and the notes it
+ * logged), in order, with its mood read off the notes and where it stopped.
+ * Labelled as written from the log, because no report step wrote it.
+ */
+export function inCharacterFromLog({ persona, notes = [], actionRecords = [], play }) {
+  const name = (persona && (persona.name || persona.slug)) || 'The persona';
+  const lines = [];
+  for (const record of actionRecords || []) {
+    if (!record || record.kind !== 'action' || !record.thought) continue;
+    const thought = String(record.thought).replace(/\s+/g, ' ').trim();
+    if (!thought) continue;
+    lines.push({ at: Number(record.actions) || 0, order: 0, text: `- Before action ${Number(record.actions) || 0} (${record.tool}${record.ok === false ? ', which did not go through' : ''}): "${clip(thought, 300)}"` });
+  }
+  for (const note of notes || []) {
+    if (!note || note.automatic || !note.text) continue;
+    // A note written on an action is recorded before that action is counted.
+    const withAction = ACTION_TOOLS.includes(note.tool);
+    const at = (Number(note.at_action) || 0) + (withAction ? 1 : 0);
+    lines.push({ at, order: 1, text: `- ${withAction ? `Note with action ${at}` : `Note after ${at} action(s)`}, ${note.kind}${note.severity ? `/${note.severity}` : ''}: "${clip(String(note.text).replace(/\s+/g, ' ').trim(), 300)}"` });
+  }
+  lines.sort((a, b) => a.at - b.at || a.order - b.order);
+  let shown = lines.map(line => line.text);
+  if (shown.length > MAX_LOG_VOICE_LINES) {
+    const half = MAX_LOG_VOICE_LINES / 2;
+    shown = [...shown.slice(0, half), `- (${shown.length - MAX_LOG_VOICE_LINES} more in the transcript)`, ...shown.slice(-half)];
+  }
+  const mood = moodFromNotes(notes);
+  const counted = Object.entries(mood.counts).filter(([, count]) => count > 0).map(([kind, count]) => `${count} ${kind}`).join(', ');
+  return [
+    '## In-character experience (written from the run log)',
+    `This section was put together by the runner from what ${name} wrote while playing, because the report step did not write one. The quoted words are the persona's own; nothing here was invented for it.`,
+    '',
+    shown.length > 0 ? shown.join('\n') : `- ${name} wrote no thoughts or notes before the run stopped.`,
+    '',
+    `**Mood, from its notes:** ${mood.words}${counted ? ` (${counted})` : ''}.`,
+    `**Where it stopped:** after ${play.actionsTaken} action(s), ${play.quitReason}${play.quitDetail ? `: ${clip(String(play.quitDetail), 200).replace(/[.\s]+$/, '')}` : ''}.`
+  ].join('\n');
+}
+
+/** The run's numbers, for a report written from the log. */
+export function dataPointsFromLog({ notes = [], screenshots = [], actions = [], play, usage = null }) {
+  const own = (notes || []).filter(note => note && !note.automatic);
+  const automatic = (notes || []).filter(note => note && note.automatic);
+  const changed = (actions || []).filter(entry => entry && entry.changed).length;
+  const spend = usage && Number.isFinite(Number(usage.total_usd)) && Number(usage.total_usd) > 0 ? `$${Number(usage.total_usd).toFixed(4)}` : null;
+  return [
+    '## Data points',
+    `- Actions taken: ${play.actionsTaken}${actions.length > 0 ? ` (${changed} of the ${actions.length} logged changed the screen)` : ''}`,
+    `- Notes the persona wrote: ${own.length}`,
+    `- Console and network notes the runner recorded: ${automatic.length}`,
+    `- Screenshots saved: ${(screenshots || []).length}`,
+    `- Stop reason: ${play.quitReason}`,
+    ...(spend ? [`- Model spend: ${spend}`] : [])
+  ].join('\n');
+}
+
+function clip(text, max) {
+  return text.length > max ? `${text.slice(0, max - 3)}...` : text;
+}
+
 /**
  * The screenshot nearest a note, by the order they happened in; a tie goes
  * to the later one, since a persona usually notes something and then
@@ -1661,10 +1851,14 @@ export function notesToFindings(notes, screenshots) {
  * to read them as one.
  */
 async function writePartialReport({ outcome, runDirectory, persona, driverName, target, notes, screenshots, actionRecords = [], play, skill, ledger, runId, jobId, startedAt, endedAt, quitReason = 'cost_cap', reason = BUDGET_PARTIAL_REASON, findings = null, preservedMarkdown = null }) {
+  // The sentiment is read off the persona's own notes (moodFromNotes), and
+  // is the scale's midpoint when it wrote none. `partial` still tells a reader
+  // that no report step judged it.
+  const mood = moodFromNotes(notes);
   const report = stampReport({
     findings: findings || notesToFindings(notes, screenshots),
-    sentiment: { fun: 3, clarity: 3, frustration: 3 },
-    would_continue: false,
+    sentiment: mood.sentiment,
+    would_continue: mood.wouldContinue,
     partial: true
   }, {
     runId,
@@ -1687,8 +1881,10 @@ async function writePartialReport({ outcome, runDirectory, persona, driverName,
     notes,
     screenshots,
     actions: actionLogFrom(actionRecords),
+    actionRecords,
     play,
-    reason
+    reason,
+    usage: ledger.toReportUsage()
   });
   const markdown = preservedMarkdown
     ? ['# Playtest (partial report draft)', '',
@@ -1962,7 +2158,7 @@ export function dropRunnerRejectedFindings(report, { notes = [], observations =
       const note = ref.match(/^\s*note\s*#?\s*(\d+)\s*$/i);
       if (observation) {
         const entry = byIndex.get(Number(observation[1]));
-        cited.push(entry && entry.driver_error ? 'driver' : 'other');
+        cited.push(entry && entry.driver_error && !gameRefusal(entry.driver_error) ? 'driver' : 'other');
       } else if (note) {
         const entry = notes[Number(note[1]) - 1];
         cited.push(entry && entry.kind === 'waiting' ? 'waiting' : entry && entry.expected_noise ? 'noise' : entry && entry.driver_error ? 'driver' : 'other');

package/src/run/regenerate.js

@@ -1,5 +1,6 @@
 import { readFile } from 'node:fs/promises';
 import path from 'node:path';
+import { gameRefusal } from './gameRefusal.js';
 import { RUN_FILES } from './paths.js';
 
 /**
@@ -81,7 +82,9 @@ export async function readTranscript(runDirectory) {
       notes.push(note);
       if (entry.tool) openNotes.push(note);
     } else if (entry.kind === 'driver_error') {
-      for (const note of callNotes) note.driver_error = true;
+      // A target the page showed as disabled was the game's answer, as the
+      // live run now records it; an older transcript is read the same way.
+      if (!gameRefusal(entry.error)) for (const note of callNotes) note.driver_error = true;
     }
     else if (entry.kind === 'screenshot' && entry.path) screenshots.push({ relativePath: entry.path, reason: entry.reason || '' });
     else if (entry.kind === 'observation') observations += 1;

package/src/run/report.js

@@ -57,6 +57,60 @@ export function lintMarkdownSections(markdown) {
   return REQUIRED_MARKDOWN_SECTIONS.filter(section => !text.includes(section));
 }
 
+/** The tool-call tags a model has been seen to leak into a written file. */
+const TOOL_TAG = '(?:parameter|invoke|content|function_calls|function_results|tool_use|tool_call|antml:[\\w-]+)';
+
+/** A line made only of tags: closers of any name, or tool-call openers. */
+const MARKUP_LINE = new RegExp(`^(?:\\s*(?:<\\/[A-Za-z][\\w:.-]*\\s*>|<${TOOL_TAG}\\b[^>]*>))+\\s*$`);
+
+/** Tool-call closers glued to the end of the last line of real text. */
+const MARKUP_SUFFIX = new RegExp(`(?:\\s*<\\/${TOOL_TAG}\\s*>)+\\s*$`);
+
+const FENCE = /^\s*(?:```|~~~)/;
+
+/**
+ * Takes tool-call markup off the end of a file the report step wrote. A real
+ * report.md ended "</content>\n</invoke>" because the model closed its tool
+ * call inside the file's content, and the runner appended its own sections
+ * after the stray tags.
+ *
+ * Only the end of the text is touched, and only outside a code fence: HTML a
+ * report quotes in backticks or in a fenced block is kept.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+export function stripTrailingToolMarkup(text) {
+  if (typeof text !== 'string' || text === '') return text;
+  const lines = text.split('\n');
+  const insideFence = count => lines.slice(0, count).filter(line => FENCE.test(line)).length % 2 === 1;
+  let end = lines.length;
+  let changed = false;
+  while (end > 0) {
+    const line = lines[end - 1];
+    if (line.trim() === '') {
+      end -= 1;
+    } else if (MARKUP_LINE.test(line) && !insideFence(end - 1)) {
+      end -= 1;
+      changed = true;
+    } else {
+      break;
+    }
+  }
+  const kept = lines.slice(0, end);
+  if (kept.length > 0) {
+    const index = kept.length - 1;
+    const last = kept[index];
+    const backticks = (last.match(/`/g) || []).length;
+    if (MARKUP_SUFFIX.test(last) && backticks % 2 === 0 && !FENCE.test(last) && !insideFence(index)) {
+      kept[index] = last.replace(MARKUP_SUFFIX, '');
+      changed = true;
+    }
+  }
+  if (!changed) return text;
+  return `${kept.join('\n').replace(/\s+$/, '')}\n`;
+}
+
 export class ReportInvalidError extends Error {
   /**
    * @param {Array<{instancePath: string, message: string}>} details