ravensight-playtest 0.7.0 → 0.7.5

package/src/run/personaLoop.js

@@ -2,16 +2,18 @@ import { rewriteMarkdown } from './dropped.js';
 import { selectAction, JEV_CONTROLLER_PROMPT } from './actionSelector.js';
 import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import path from 'node:path';
-import { prepareMessages, MAX_IMAGES_IN_CONTEXT, KEEP_OBSERVATIONS, shrinkImages } from './context.js';
+import { prepareMessages, repairLastTurn, MAX_IMAGES_IN_CONTEXT, KEEP_OBSERVATIONS, shrinkImages } from './context.js';
 import { withRollingCacheBreakpoint } from './fixLoop.js';
 import { callStepWithRetry, ProxyError, STEPS } from './model.js';
 import { RUN_FILES } from './paths.js';
-import { appendNormalisedMarkdown, checkEvidence, createReportValidator, lintMarkdownSections, normaliseEnums, normaliseEvidence, repairInstruction, REQUIRED_MARKDOWN_SECTIONS, stampReport, writeReportFiles } from './report.js';
+import { reserveDecision } from './budgetReserve.js';
+import { appendNormalisedMarkdown, checkEvidence, createReportValidator, lintMarkdownSections, normaliseEnums, normaliseEvidence, repairInstruction, REQUIRED_MARKDOWN_SECTIONS, stampReport, stripTrailingToolMarkup, writeReportFiles } from './report.js';
 import { assertNoSecrets, scanForSecrets } from './secretScan.js';
 import { ACTION_TOOLS, PLAY_TOOLS, SWIPE_TOOL, REPORT_TOOLS, executeTool } from './tools.js';
 import { createTranscript } from './transcript.js';
 import { createUsageLedger, writeUsageFile } from './usage.js';
 import { fillPlaceholders, fillReportPlaceholders } from './placeholders.js';
+import { gameRefusal } from './gameRefusal.js';
 import { BUILT_IN_NOISE, expectedNoiseForPrompt, isExpectedNoise } from './noise.js';
 import { MIN_PASSWORD_CHARS } from '../briefLocal.js';
 
@@ -74,6 +76,25 @@ const ABOVE_MINOR = Object.freeze(['blocker', 'major']);
 /** What a capped finding's description gains, so the reader knows why. */
 export const NETWORK_CAP_NOTE = 'Severity capped at minor by the runner: the only evidence is automatic network notes, and a failed request is a finding above minor only when the persona saw the game break because of it.';
 
+/** How each way of leaving the build reads in a note. */
+const OFF_SITE_VIA = Object.freeze({
+  navigation: 'a link, form or script on the page',
+  'new window': 'a new window',
+  redirect: 'a redirect',
+  navigate: 'the persona\'s navigate'
+});
+
+/**
+ * The network note for one attempt to leave the build.
+ *
+ * @param {{host?: string, via?: string}} attempt
+ * @returns {string}
+ */
+export function offSiteNoteText(attempt) {
+  const via = OFF_SITE_VIA[attempt && attempt.via] || 'the page';
+  return `off-site: ${via} tried to leave the build for ${(attempt && attempt.host) || 'another site'}; the runner kept the page on the build.`;
+}
+
 /** A persona note kind as a report finding category. */
 // `waiting` has no category on purpose: a "waiting for X to finish" note is
 // context, and nothing turns it into a finding.
@@ -102,6 +123,31 @@ export const ACT_MAX_TOKENS = 1024;
 /** Token ceiling asked for when writing the report. */
 export const REPORT_MAX_TOKENS = 4096;
 
+/** The most characters of a persona's closing message kept as its quit detail. */
+export const MAX_QUIT_DETAIL_CHARS = 160;
+
+/**
+ * A persona's closing message as a one-line quit detail: its first sentence,
+ * without markdown, cut to MAX_QUIT_DETAIL_CHARS. A message that is already
+ * report text says only that the persona stopped.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+export function personaQuitDetail(text) {
+  const plain = 'the persona stopped playing';
+  const first = String(text || '').split(/\n/).map(line => line.replace(/[*_`#>]+/g, '').trim()).find(Boolean) || '';
+  if (!first || /report\.(md|json)|write_file|\b(?:writ(?:e|ing)|start(?:ing)? on) (?:the |my )?(?:final )?report\b/i.test(first)) return plain;
+  const sentence = (first.match(/^.*?[.!?](?=\s|$)/) || [first])[0];
+  if (sentence.length <= MAX_QUIT_DETAIL_CHARS) return sentence;
+  const cut = sentence.slice(0, MAX_QUIT_DETAIL_CHARS - 3);
+  const space = cut.lastIndexOf(' ');
+  return `${space > 40 ? cut.slice(0, space) : cut}...`;
+}
+
+/** The stop detail when play ends to keep the report's budget. */
+export const RESERVE_STOP_DETAIL = 'stopped early so the rest of the model budget pays for the report';
+
 /**
  * @typedef {Object} PersonaRunOutcome
  * @property {string} state one of succeeded, failed, canceled, budget_exceeded
@@ -373,6 +419,11 @@ async function playLoop({
   heartbeat,
   onProgress = () => {},
   onBudgetWarning = null,
+  // The run's model budget in USD, from register (`limits.model_usd`). With it
+  // the loop stops early enough that the report still fits; without it (an
+  // older server) only the server's 80 percent warning does.
+  modelBudgetUsd = null,
+  reportSkill = null,
   log = () => {}
 }) {
   const system = buildSystem({ skill, persona, brief, localBrief, intentNotes, driverName, target, maxActions, runId });
@@ -406,9 +457,39 @@ async function playLoop({
   let budgetWarned = false;
   let budgetExceeded = false;
   let reportBudgetAvailable = false;
+  let reserveStopped = false;
   let canceled = false;
   let unchanged = 0;
   const maxTurns = maxActions * 3 + 20;
+  // What the last few act turns really cost, in weighted units.
+  const recentActUnits = [];
+
+  /**
+   * Whether the next act turn would leave too little for the report. Sized
+   * from the act request about to be sent and the report request as it would
+   * be built now; see ./budgetReserve.js.
+   */
+  const reserveCheck = () => {
+    const spend = modelBudgetUsd ? ledger.spendEstimate() : null;
+    if (!spend) return { stop: false, known: false, figures: {} };
+    const prepared = withRollingCacheBreakpoint(prepareMessages(messages, { maxImages: imageBudget, keepObservations }));
+    const actChars = JSON.stringify({ messages: prepared, max_tokens: ACT_MAX_TOKENS, stream: true, system, tools: playTools }).length;
+    const reportRequest = buildReportRequest({
+      skill: reportSkill || skill, persona, brief, localBrief, intentNotes, driverName, target, runId, jobId,
+      play: { actionsTaken: actions, quitReason: 'budget_exhausted', quitDetail: RESERVE_STOP_DETAIL, actionLog: actionLogFrom(actionRecords) },
+      notes, observations, screenshots
+    });
+    const reportChars = JSON.stringify({ messages: reportRequest.messages, max_tokens: REPORT_MAX_TOKENS, stream: true, system: reportRequest.system, tools: REPORT_TOOLS }).length;
+    return reserveDecision({
+      usdMax: modelBudgetUsd,
+      spend,
+      actChars,
+      actMaxTokens: ACT_MAX_TOKENS,
+      actCostUnits: recentActUnits.length > 0 ? Math.max(...recentActUnits) : 0,
+      reportChars,
+      reportMaxTokens: REPORT_MAX_TOKENS
+    });
+  };
 
   // Notes waiting for the next observation, which is the screen they are
   // checked against: a failure the next screen does not show is not a
@@ -454,26 +535,53 @@ async function playLoop({
       quitDetail = 'the job was cancelled';
       break;
     }
+    if (turns > 0) {
+      const reserve = reserveCheck();
+      if (reserve.stop) {
+        reserveStopped = true;
+        quitReason = 'budget_exhausted';
+        quitDetail = RESERVE_STOP_DETAIL;
+        transcript.write({ kind: 'budget_reserve_stop', ...reserve.figures });
+        log(`stopping play after ${actions} action(s) so the rest of the model budget pays for the report (spent about $${reserve.figures.usd_spent} of $${reserve.figures.usd_max}; the report needs about $${reserve.figures.report_reserve_usd}).`);
+        break;
+      }
+    }
     turns += 1;
     let answer;
     const askedAt = Date.now();
+    const askAct = () => callWithImageRetry({
+      client,
+      jobId,
+      runId,
+      step: STEPS.personaAct,
+      system,
+      messages,
+      tools: playTools,
+      maxTokens: ACT_MAX_TOKENS,
+      escalate: shouldEscalate(persona),
+      imageBudget,
+      keepObservations,
+      onImageBudget: next => {
+        imageBudget = next;
+      }
+    });
     try {
-      answer = await callWithImageRetry({
-        client,
-        jobId,
-        runId,
-        step: STEPS.personaAct,
-        system,
-        messages,
-        tools: playTools,
-        maxTokens: ACT_MAX_TOKENS,
-        escalate: shouldEscalate(persona),
-        imageBudget,
-        keepObservations,
-        onImageBudget: next => {
-          imageBudget = next;
-        }
-      });
+      try {
+        answer = await askAct();
+      } catch (error) {
+        // The provider refused the body. Every tool_result is already made
+        // valid before it is sent (./context.js), so this is a shape nobody
+        // has seen yet: the newest turn is rewritten as plain text and asked
+        // once more. A second refusal ends play, and the run still reports
+        // from its step log (runPersona's error path), because a run that
+        // played never comes back empty.
+        if (!(error instanceof ProxyError) || error.code !== 'upstream_rejected') throw error;
+        log(`the model provider rejected this turn (${error.message}); retrying once with the last turn as plain text.`);
+        transcript.write({ kind: 'provider_rejected', retry: true, code: error.code, provider_type: error.figures.provider_type || null, provider_message: error.figures.provider_message || null });
+        const repaired = repairLastTurn(messages);
+        messages.splice(0, messages.length, ...repaired);
+        answer = await askAct();
+      }
     } catch (error) {
       if (error instanceof ProxyError && error.isBudget) {
         budgetExceeded = true;
@@ -491,6 +599,8 @@ async function playLoop({
     }
 
     ledger.record({ model: answer.model, usage: answer.usage, ms: Date.now() - askedAt });
+    recentActUnits.push(ledger.lastCallUnits || 0);
+    if (recentActUnits.length > 3) recentActUnits.shift();
     // The warn turn is a turn the developer paid for, so its tool calls are
     // executed like any other and the wrap-up notice rides along on that
     // turn's tool_result. Two things this avoids: discarding a paid action,
@@ -513,7 +623,12 @@ async function playLoop({
       // No tool call is how a persona stops: there is no quit tool in the
       // step's allowlist, and adding one would be refused by the proxy.
       quitReason = warnedNow ? 'budget_exhausted' : 'persona_quit';
-      quitDetail = answer.text.slice(0, 400);
+      // The whole closing message is in the transcript. The detail is one
+      // short line: it becomes report.json's quit_detail, and a closing turn
+      // is often the persona starting on its report ("Now writing the final
+      // report. **report.md:** ..."), which in 0.7.1 ended up as a
+      // succeeded run's reason on the dashboard.
+      quitDetail = personaQuitDetail(answer.text);
       transcript.write({ kind: 'quit', reason: quitReason, text: answer.text });
       break;
     }
@@ -583,6 +698,7 @@ async function playLoop({
         ok: !executed.error,
         error: executed.error,
         changed: executed.changed,
+        ...(executed.refused ? { refused: executed.refused } : {}),
         actions
       };
       transcript.write(actionLine);
@@ -611,6 +727,13 @@ async function playLoop({
         // declared wedged for doing the right thing.
         if (ACTION_TOOLS.includes(toolUse.name)) unchanged = executed.changed ? 0 : unchanged + 1;
       }
+      if (executed.refused) {
+        // A refused off-site navigation is a step that changed nothing: it
+        // counts toward the stuck limit like any other, and play goes on.
+        unchanged += 1;
+        recentActions.push(`${toolUse.name}: refused, off-site`);
+        if (recentActions.length > 10) recentActions.shift();
+      }
       onProgress({ phase: 'playing', persona: persona.slug, actions, maxActions, tool: toolUse.name });
     }
 
@@ -652,6 +775,24 @@ async function playLoop({
       asides.push(`The game logged ${consoleErrors.length} error(s): ${consoleErrors.map(entry => entry.text).join(' | ').slice(0, 800)}${shot ? ` A screenshot is saved at ${shot.relativePath}.` : ''}`);
     }
 
+    // Every attempt to leave the build this turn (a clicked link, a form, a
+    // new window, a script or a server redirect, or a refused navigate) is a
+    // network note: the runner kept the page on the build, and the report
+    // step reads it as context. A persona told "Refused" already knows; a
+    // timer redirect between actions is only ever seen here.
+    const leaving = typeof driver.offSiteAttempts === 'function' ? driver.offSiteAttempts() : [];
+    for (const attempt of leaving) {
+      const note = {
+        kind: 'network',
+        text: offSiteNoteText(attempt),
+        automatic: true,
+        off_site: true,
+        url: String(attempt.url || attempt.host || '').slice(0, 200)
+      };
+      noted(note);
+      transcript.write({ kind: 'note', note: scrubSecrets(note, secrets) });
+    }
+
     const wedged = unchanged >= STUCK_LIMIT;
     if (!wedged && unchanged > 0 && unchanged % STUCK_HINT_AFTER === 0) {
       transcript.write({ kind: 'stuck', unchanged, fatal: false });
@@ -695,6 +836,7 @@ async function playLoop({
     budgetWarned,
     budgetExceeded,
     reportBudgetAvailable,
+    reserveStopped,
     canceled,
     imageBudget,
     finalScreenshot: final ? final.relativePath : null,
@@ -733,55 +875,7 @@ async function reportPhase({
   // The report step never sees the test account: it has nothing to sign in
   // to, and the password crosses the model proxy during play only.
   const secrets = secretsOf(localBrief);
-  const reportLocal = localBrief && Array.isArray(localBrief.expected_noise) ? { expected_noise: localBrief.expected_noise } : {};
-  const system = buildSystem({ skill, persona, brief, localBrief: reportLocal, intentNotes, driverName, target, maxActions: play.actionsTaken, runId });
-  const messages = [{
-    role: 'user',
-    content: [{
-      type: 'text',
-      text: [
-        `The session is over. Write report.md and report.json for run ${runId} of job ${jobId}.`,
-        `persona: ${persona.slug}   driver: ${driverName}   target: ${target}`,
-        `actions taken: ${play.actionsTaken}   stop reason: ${play.quitReason}${play.quitDetail ? ` (${play.quitDetail})` : ''}`,
-        '',
-        'Notes you logged, in order:',
-        notes.length > 0 ? notes.map((note, i) => noteLine(note, i)).join('\n') : '(none)',
-        '',
-        'Notes marked automatic were recorded by the runner from the console, with no severity: decide one from what the persona saw. Network notes are context, not bugs: a failed request is a finding only when the persona saw a broken screen because of it. A finding whose only evidence is automatic network notes is capped at minor by the runner.',
-        'Driver errors are never evidence of a game bug. An observation marked [driver error: ...] or a note "written on a driver error" means the runner could not carry out the action (a target that matched two elements, locator code, a ref not on screen, a timeout on a page that did not change); the game did nothing wrong there. Never report a blocked loop, a frozen screen or a broken button from them. The runner drops any blocker or major whose only cited observations are driver errors.',
-        `Notes marked expected noise match the brief's expected_noise or the built in rule (${BUILT_IN_NOISE}); they are never a finding of any severity, and the runner drops a finding whose transcript evidence is only expected noise.`,
-        'Notes of kind waiting ("waiting for X to finish") are context, never a finding of any severity, and the runner drops a finding cited only by them. Before reporting a loading problem, check the later observations and screenshots: if they show the content loaded, it is not a finding.',
-        '',
-        'Observations the persona was shown, by index:',
-        observationLines(observations),
-        '',
-        'Actions you took (every action that went through, numbered by action; "screen changed" means the screen after it differed from the screen before):',
-        actionLogLines(play.actionLog),
-        '',
-        'A claim in report.md or report.json that something was never done, never worked, did nothing or could not be done must agree with this list. If the list shows you clicked a control and the screen changed, do not write that you never clicked it or that it did nothing; describe what the screen showed after it instead. The runner downgrades a finding that says otherwise.',
-        '',
-        'Network notes (failed requests the runner recorded, with the URL and the status; list them under a "Network notes" heading in report.md when there are any):',
-        networkNoteLines(notes),
-        '',
-        'Each note carries "after": the URL and title of the observation that followed it. A blocker or major finding must cite the observation index where the failure is visible, as evidence {"type": "transcript_step", "ref": "observation N"}, and must not be contradicted by its note\'s "after" state or by a later persona note: if the next screen or a later note shows the game working, it is not a blocker or a major. Cite a note as {"type": "transcript_step", "ref": "note N"}.',
-        'A blocker or major finding must also cite the screenshot taken at or after that observation (the list below says which observation each one followed), and that screenshot must not contradict it: if it shows the thing the finding says is missing or broken, such as the field or the button, it is not a blocker or a major. A text observation can be cut short on a long page; the screenshot is what the player saw. After you write the report, the runner shows each blocker or major (at most three) its cited screenshot and downgrades one step any finding the image contradicts.',
-        '',
-        'Every evidence entry is an object with exactly two keys, "type" and "ref", and nothing else. A screenshot is {"type": "screenshot", "ref": "screenshots/04-example.png"}, never "path" or "file".',
-        '',
-        'Never write calendar dates or times in the prose of report.md or report.json. The runner stamps when the report was generated, so leave the template\'s Date out rather than writing a placeholder for it.',
-        'Replace every <...> slot in the template with real content. Never copy a <...> placeholder into the report.',
-        '',
-        'Screenshots available as evidence (use these exact paths in evidence refs):',
-        screenshots.length > 0 ? screenshots.map(screenshotLine).join('\n') : '(none)',
-        '',
-        `report.md must contain these section headings, spelled exactly: ${REQUIRED_MARKDOWN_SECTIONS.map(section => `"${section}"`).join(', ')}.`,
-        'Write report.md first, then report.json. Leave usage as zeros; the runner fills it in.'
-      ].join('\n')
-    }]
-  }];
-  // The prompt is built from notes, observations and the action log; none
-  // of them may carry the test account password into the report step.
-  messages[0].content[0].text = scrubSecrets(messages[0].content[0].text, secrets);
+  const { system, messages } = buildReportRequest({ skill, persona, brief, localBrief, intentNotes, driverName, target, runId, jobId, play, notes, observations, screenshots });
 
   let report = null;
   let markdown = null;
@@ -808,7 +902,7 @@ async function reportPhase({
   // never escalates to a dearer tier after a warning, and a 402 on a report
   // turn gets one retry with the output ceiling cut to what the refusal says
   // still fits.
-  const escalate = shouldEscalate(persona) && !play.budgetWarned && !play.budgetExceeded;
+  const escalate = shouldEscalate(persona) && !play.budgetWarned && !play.budgetExceeded && !play.reserveStopped;
   let maxTokens = REPORT_MAX_TOKENS;
   let shrunk = false;
   let lastCandidate = null;
@@ -868,7 +962,9 @@ async function reportPhase({
       if (toolUse.name !== 'write_file' || toolUse.malformed) continue;
       const name = String(toolUse.input.path || '');
       if (name !== RUN_FILES.reportMd && name !== RUN_FILES.reportJson) continue;
-      files.set(name, String(toolUse.input.content || ''));
+      // A model sometimes closes its tool call inside the file's content;
+      // the stray tags come off before the file is validated or written.
+      files.set(name, stripTrailingToolMarkup(String(toolUse.input.content || '')));
     }
     if (files.has(RUN_FILES.reportMd)) markdown = files.get(RUN_FILES.reportMd);
     if (files.has(RUN_FILES.reportJson)) jsonText = files.get(RUN_FILES.reportJson);
@@ -1168,6 +1264,60 @@ export async function checkClaimsAgainstScreenshots({ client, jobId, runId, repo
   return { report: { ...report, findings }, downgraded };
 }
 
+/**
+ * The report step's request: its system prompt and its one user message. Built
+ * the same way for the call and for sizing it during play (./budgetReserve.js),
+ * so the budget kept back for the report is the report's own size.
+ *
+ * @returns {{system: Array, messages: Array}}
+ */
+export function buildReportRequest({ skill, persona, brief, localBrief = {}, intentNotes, driverName, target, runId, jobId, play, notes, observations = [], screenshots }) {
+  const secrets = secretsOf(localBrief);
+  const reportLocal = localBrief && Array.isArray(localBrief.expected_noise) ? { expected_noise: localBrief.expected_noise } : {};
+  const system = buildSystem({ skill, persona, brief, localBrief: reportLocal, intentNotes, driverName, target, maxActions: play.actionsTaken, runId });
+  const text = [
+    `The session is over. Write report.md and report.json for run ${runId} of job ${jobId}.`,
+    `persona: ${persona.slug}   driver: ${driverName}   target: ${target}`,
+    `actions taken: ${play.actionsTaken}   stop reason: ${play.quitReason}${play.quitDetail ? ` (${play.quitDetail})` : ''}`,
+    '',
+    'Notes you logged, in order:',
+    notes.length > 0 ? notes.map((note, i) => noteLine(note, i)).join('\n') : '(none)',
+    '',
+    'Notes marked automatic were recorded by the runner from the console, with no severity: decide one from what the persona saw. Network notes are context, not bugs: a failed request is a finding only when the persona saw a broken screen because of it. A finding whose only evidence is automatic network notes is capped at minor by the runner.',
+    'Driver errors are never evidence of a game bug. An observation marked [driver error: ...] or a note "written on a driver error" means the runner could not carry out the action (a target that matched two elements, locator code, a ref not on screen, a timeout on a page that did not change); the game did nothing wrong there. Never report a blocked loop, a frozen screen or a broken button from them. The runner drops any blocker or major whose only cited observations are driver errors.',
+    `Notes marked expected noise match the brief's expected_noise or the built in rule (${BUILT_IN_NOISE}); they are never a finding of any severity, and the runner drops a finding whose transcript evidence is only expected noise.`,
+    'Notes of kind waiting ("waiting for X to finish") are context, never a finding of any severity, and the runner drops a finding cited only by them. Before reporting a loading problem, check the later observations and screenshots: if they show the content loaded, it is not a finding.',
+    '',
+    'Observations the persona was shown, by index:',
+    observationLines(observations),
+    '',
+    'Actions you took (every action that went through, numbered by action; "screen changed" means the screen after it differed from the screen before):',
+    actionLogLines(play.actionLog),
+    '',
+    'A claim in report.md or report.json that something was never done, never worked, did nothing or could not be done must agree with this list. If the list shows you clicked a control and the screen changed, do not write that you never clicked it or that it did nothing; describe what the screen showed after it instead. The runner downgrades a finding that says otherwise.',
+    '',
+    'Network notes (failed requests the runner recorded, with the URL and the status; list them under a "Network notes" heading in report.md when there are any):',
+    networkNoteLines(notes),
+    '',
+    'Each note carries "after": the URL and title of the observation that followed it. A blocker or major finding must cite the observation index where the failure is visible, as evidence {"type": "transcript_step", "ref": "observation N"}, and must not be contradicted by its note\'s "after" state or by a later persona note: if the next screen or a later note shows the game working, it is not a blocker or a major. Cite a note as {"type": "transcript_step", "ref": "note N"}.',
+    'A blocker or major finding must also cite the screenshot taken at or after that observation (the list below says which observation each one followed), and that screenshot must not contradict it: if it shows the thing the finding says is missing or broken, such as the field or the button, it is not a blocker or a major. A text observation can be cut short on a long page; the screenshot is what the player saw. After you write the report, the runner shows each blocker or major (at most three) its cited screenshot and downgrades one step any finding the image contradicts.',
+    '',
+    'Every evidence entry is an object with exactly two keys, "type" and "ref", and nothing else. A screenshot is {"type": "screenshot", "ref": "screenshots/04-example.png"}, never "path" or "file".',
+    '',
+    'Never write calendar dates or times in the prose of report.md or report.json. The runner stamps when the report was generated, so leave the template\'s Date out rather than writing a placeholder for it.',
+    'Replace every <...> slot in the template with real content. Never copy a <...> placeholder into the report.',
+    '',
+    'Screenshots available as evidence (use these exact paths in evidence refs):',
+    screenshots.length > 0 ? screenshots.map(screenshotLine).join('\n') : '(none)',
+    '',
+    `report.md must contain these section headings, spelled exactly: ${REQUIRED_MARKDOWN_SECTIONS.map(section => `"${section}"`).join(', ')}.`,
+    'Write report.md first, then report.json. Leave usage as zeros; the runner fills it in.'
+  ].join('\n');
+  // The prompt is built from notes, observations and the action log; none
+  // of them may carry the test account password into the report step.
+  return { system, messages: [{ role: 'user', content: [{ type: 'text', text: scrubSecrets(text, secrets) }] }] };
+}
+
 /** The smallest output ceiling worth retrying a report turn with. */
 export const REPORT_MIN_TOKENS = 1536;
 
@@ -1553,7 +1703,7 @@ async function finish(outcome, ledger, runDirectory, transcript, heartbeat) {
  * screenshot list and says plainly that it is partial, rather than
  * inventing a persona verdict nobody gave.
  */
-export function renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, actions = [], play, reason }) {
+export function renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, actions = [], actionRecords = [], play, reason, usage = null }) {
   const good = notes.filter(note => note.kind === 'good');
   const bad = notes.filter(note => note.kind === 'bug' || note.kind === 'confusing');
   const network = notes.filter(note => note.automatic && note.url);
@@ -1565,13 +1715,17 @@ export function renderFallbackMarkdown({ persona, driverName, target, notes, scr
   // report. A partial report is still a report somebody reads, and a lint
   // finding on it would be noise about a run that already went wrong.
   return [
-    `# Playtest: ${persona.name || persona.slug} (partial)`,
+    `# Playtest: ${persona.name || persona.slug} (partial, written from the run log)`,
     `**Driver:** ${driverName} - **Target:** ${target}`,
     `**Session length:** ${play.actionsTaken} actions / ${play.quitReason}`,
     '',
     '## Why this report is partial',
     reason,
     '',
+    inCharacterFromLog({ persona, notes, actionRecords, play }),
+    '',
+    dataPointsFromLog({ notes, screenshots, actions, play, usage }),
+    '',
     ...(actions.length > 0 ? ['## What the persona did', actionLogLines(actions), ''] : []),
     '## What worked',
     list(good),
@@ -1592,6 +1746,103 @@ export function renderFallbackMarkdown({ persona, driverName, target, notes, scr
   ].join('\n');
 }
 
+/** The most lines of the persona's own words a log-written section quotes. */
+export const MAX_LOG_VOICE_LINES = 16;
+
+/**
+ * The persona's mood, read off its own notes: what a report written from the
+ * log says it felt, and the sentiment report.json carries. Only the notes the
+ * persona wrote count: an automatic console note is not a feeling, and a note
+ * written on a driver error is the runner's failure.
+ *
+ * @param {Array<Object>} notes
+ * @returns {{counts: Object, sentiment: {fun: number, clarity: number, frustration: number}, wouldContinue: boolean, words: string, known: boolean}}
+ */
+export function moodFromNotes(notes = []) {
+  const own = (notes || []).filter(note => note && !note.automatic && !note.driver_error && !note.expected_noise);
+  const counts = { good: 0, bug: 0, confusing: 0, suggestion: 0, reaction: 0, waiting: 0 };
+  for (const note of own) if (note.kind in counts) counts[note.kind] += 1;
+  const clamp = value => Math.min(5, Math.max(1, Math.round(value)));
+  const troubles = counts.bug + counts.confusing;
+  const known = counts.good + troubles + counts.reaction + counts.suggestion > 0;
+  if (!known) {
+    return { counts, sentiment: { fun: 3, clarity: 3, frustration: 3 }, wouldContinue: false, words: 'unknown: the persona logged no notes about how it was going', known };
+  }
+  const sentiment = {
+    fun: clamp(3 + counts.good - counts.bug - counts.confusing / 2),
+    clarity: clamp(4 - counts.confusing - counts.bug / 2),
+    frustration: clamp(1 + counts.bug + counts.confusing)
+  };
+  const words = counts.good > troubles
+    ? 'mostly positive'
+    : troubles > counts.good
+      ? (counts.good > 0 ? 'mixed, leaning frustrated' : 'mostly frustrated or confused')
+      : 'mixed';
+  return { counts, sentiment, wouldContinue: counts.good > troubles, words, known };
+}
+
+/**
+ * The in-character section of a report written from the log: the persona's
+ * own words while it played (the thought before each action and the notes it
+ * logged), in order, with its mood read off the notes and where it stopped.
+ * Labelled as written from the log, because no report step wrote it.
+ */
+export function inCharacterFromLog({ persona, notes = [], actionRecords = [], play }) {
+  const name = (persona && (persona.name || persona.slug)) || 'The persona';
+  const lines = [];
+  for (const record of actionRecords || []) {
+    if (!record || record.kind !== 'action' || !record.thought) continue;
+    const thought = String(record.thought).replace(/\s+/g, ' ').trim();
+    if (!thought) continue;
+    lines.push({ at: Number(record.actions) || 0, order: 0, text: `- Before action ${Number(record.actions) || 0} (${record.tool}${record.ok === false ? ', which did not go through' : ''}): "${clip(thought, 300)}"` });
+  }
+  for (const note of notes || []) {
+    if (!note || note.automatic || !note.text) continue;
+    // A note written on an action is recorded before that action is counted.
+    const withAction = ACTION_TOOLS.includes(note.tool);
+    const at = (Number(note.at_action) || 0) + (withAction ? 1 : 0);
+    lines.push({ at, order: 1, text: `- ${withAction ? `Note with action ${at}` : `Note after ${at} action(s)`}, ${note.kind}${note.severity ? `/${note.severity}` : ''}: "${clip(String(note.text).replace(/\s+/g, ' ').trim(), 300)}"` });
+  }
+  lines.sort((a, b) => a.at - b.at || a.order - b.order);
+  let shown = lines.map(line => line.text);
+  if (shown.length > MAX_LOG_VOICE_LINES) {
+    const half = MAX_LOG_VOICE_LINES / 2;
+    shown = [...shown.slice(0, half), `- (${shown.length - MAX_LOG_VOICE_LINES} more in the transcript)`, ...shown.slice(-half)];
+  }
+  const mood = moodFromNotes(notes);
+  const counted = Object.entries(mood.counts).filter(([, count]) => count > 0).map(([kind, count]) => `${count} ${kind}`).join(', ');
+  return [
+    '## In-character experience (written from the run log)',
+    `This section was put together by the runner from what ${name} wrote while playing, because the report step did not write one. The quoted words are the persona's own; nothing here was invented for it.`,
+    '',
+    shown.length > 0 ? shown.join('\n') : `- ${name} wrote no thoughts or notes before the run stopped.`,
+    '',
+    `**Mood, from its notes:** ${mood.words}${counted ? ` (${counted})` : ''}.`,
+    `**Where it stopped:** after ${play.actionsTaken} action(s), ${play.quitReason}${play.quitDetail ? `: ${clip(String(play.quitDetail), 200).replace(/[.\s]+$/, '')}` : ''}.`
+  ].join('\n');
+}
+
+/** The run's numbers, for a report written from the log. */
+export function dataPointsFromLog({ notes = [], screenshots = [], actions = [], play, usage = null }) {
+  const own = (notes || []).filter(note => note && !note.automatic);
+  const automatic = (notes || []).filter(note => note && note.automatic);
+  const changed = (actions || []).filter(entry => entry && entry.changed).length;
+  const spend = usage && Number.isFinite(Number(usage.total_usd)) && Number(usage.total_usd) > 0 ? `$${Number(usage.total_usd).toFixed(4)}` : null;
+  return [
+    '## Data points',
+    `- Actions taken: ${play.actionsTaken}${actions.length > 0 ? ` (${changed} of the ${actions.length} logged changed the screen)` : ''}`,
+    `- Notes the persona wrote: ${own.length}`,
+    `- Console and network notes the runner recorded: ${automatic.length}`,
+    `- Screenshots saved: ${(screenshots || []).length}`,
+    `- Stop reason: ${play.quitReason}`,
+    ...(spend ? [`- Model spend: ${spend}`] : [])
+  ].join('\n');
+}
+
+function clip(text, max) {
+  return text.length > max ? `${text.slice(0, max - 3)}...` : text;
+}
+
 /**
  * The screenshot nearest a note, by the order they happened in; a tie goes
  * to the later one, since a persona usually notes something and then
@@ -1661,10 +1912,14 @@ export function notesToFindings(notes, screenshots) {
  * to read them as one.
  */
 async function writePartialReport({ outcome, runDirectory, persona, driverName, target, notes, screenshots, actionRecords = [], play, skill, ledger, runId, jobId, startedAt, endedAt, quitReason = 'cost_cap', reason = BUDGET_PARTIAL_REASON, findings = null, preservedMarkdown = null }) {
+  // The sentiment is read off the persona's own notes (moodFromNotes), and
+  // is the scale's midpoint when it wrote none. `partial` still tells a reader
+  // that no report step judged it.
+  const mood = moodFromNotes(notes);
   const report = stampReport({
     findings: findings || notesToFindings(notes, screenshots),
-    sentiment: { fun: 3, clarity: 3, frustration: 3 },
-    would_continue: false,
+    sentiment: mood.sentiment,
+    would_continue: mood.wouldContinue,
     partial: true
   }, {
     runId,
@@ -1687,8 +1942,10 @@ async function writePartialReport({ outcome, runDirectory, persona, driverName,
     notes,
     screenshots,
     actions: actionLogFrom(actionRecords),
+    actionRecords,
     play,
-    reason
+    reason,
+    usage: ledger.toReportUsage()
   });
   const markdown = preservedMarkdown
     ? ['# Playtest (partial report draft)', '',
@@ -1962,7 +2219,7 @@ export function dropRunnerRejectedFindings(report, { notes = [], observations =
       const note = ref.match(/^\s*note\s*#?\s*(\d+)\s*$/i);
       if (observation) {
         const entry = byIndex.get(Number(observation[1]));
-        cited.push(entry && entry.driver_error ? 'driver' : 'other');
+        cited.push(entry && entry.driver_error && !gameRefusal(entry.driver_error) ? 'driver' : 'other');
       } else if (note) {
         const entry = notes[Number(note[1]) - 1];
         cited.push(entry && entry.kind === 'waiting' ? 'waiting' : entry && entry.expected_noise ? 'noise' : entry && entry.driver_error ? 'driver' : 'other');

package/src/run/regenerate.js

@@ -1,5 +1,6 @@
 import { readFile } from 'node:fs/promises';
 import path from 'node:path';
+import { gameRefusal } from './gameRefusal.js';
 import { RUN_FILES } from './paths.js';
 
 /**
@@ -81,7 +82,9 @@ export async function readTranscript(runDirectory) {
       notes.push(note);
       if (entry.tool) openNotes.push(note);
     } else if (entry.kind === 'driver_error') {
-      for (const note of callNotes) note.driver_error = true;
+      // A target the page showed as disabled was the game's answer, as the
+      // live run now records it; an older transcript is read the same way.
+      if (!gameRefusal(entry.error)) for (const note of callNotes) note.driver_error = true;
     }
     else if (entry.kind === 'screenshot' && entry.path) screenshots.push({ relativePath: entry.path, reason: entry.reason || '' });
     else if (entry.kind === 'observation') observations += 1;