ravensight-playtest 0.2.8 → 0.3.0

package/src/run/personaLoop.js

@@ -1,12 +1,12 @@
+import { rewriteMarkdown } from './dropped.js';
 import { selectAction, JEV_CONTROLLER_PROMPT } from './actionSelector.js';
 import { mkdir, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import { prepareMessages, MAX_IMAGES_IN_CONTEXT, KEEP_OBSERVATIONS, shrinkImages } from './context.js';
-import { observationHash } from './drivers/driver.js';
 import { withRollingCacheBreakpoint } from './fixLoop.js';
 import { callStepWithRetry, ProxyError, STEPS } from './model.js';
 import { RUN_FILES } from './paths.js';
-import { checkEvidence, createReportValidator, lintMarkdownSections, repairInstruction, REQUIRED_MARKDOWN_SECTIONS, stampReport, writeReportFiles } from './report.js';
+import { checkEvidence, createReportValidator, lintMarkdownSections, normaliseEvidence, repairInstruction, REQUIRED_MARKDOWN_SECTIONS, stampReport, writeReportFiles } from './report.js';
 import { assertNoSecrets } from './secretScan.js';
 import { ACTION_TOOLS, PLAY_TOOLS, SWIPE_TOOL, REPORT_TOOLS, executeTool } from './tools.js';
 import { createTranscript } from './transcript.js';
@@ -70,6 +70,8 @@ const ABOVE_MINOR = Object.freeze(['blocker', 'major']);
 export const NETWORK_CAP_NOTE = 'Severity capped at minor by the runner: the only evidence is automatic network notes, and a failed request is a finding above minor only when the persona saw the game break because of it.';
 
 /** A persona note kind as a report finding category. */
+// `waiting` has no category on purpose: a "waiting for X to finish" note is
+// context, and nothing turns it into a finding.
 export const NOTE_CATEGORY = Object.freeze({ bug: 'bug', confusing: 'ux', suggestion: 'ux', good: 'ux', reaction: 'ux' });
 
 /** Stop injecting the "nothing changed" hint after this many identical screens. */
@@ -78,8 +80,16 @@ export const STUCK_HINT_AFTER = 5;
 /** Give up entirely after this many identical screens: the persona is wedged. */
 export const STUCK_LIMIT = 12;
 
-/** Turns allowed after a budget warning, to land the report. */
-export const WRAP_UP_TURNS = 3;
+/**
+ * Act turns allowed after a budget warning: none. The proxy reserves each
+ * call's pessimistic projection (the whole request at the full input rate,
+ * plus the output ceiling) before it sends it, so every act turn after the
+ * warning ate into the 20 percent the report step needs, and in the Texas run
+ * three of them left too little for the report's own reservation: the report
+ * call was refused with 402 and the run ended at cost_cap with no report.
+ * The warned turn's own actions are still played; then the report is written.
+ */
+export const WRAP_UP_TURNS = 0;
 
 /** Token ceiling asked for per act turn. The proxy clamps it to the step's own. */
 export const ACT_MAX_TOKENS = 1024;
@@ -331,6 +341,7 @@ async function playLoop({
   keepObservations,
   heartbeat,
   onProgress = () => {},
+  onBudgetWarning = null,
   log = () => {}
 }) {
   const system = buildSystem({ skill, persona, brief, intentNotes, driverName, target, maxActions });
@@ -362,7 +373,6 @@ async function playLoop({
   let budgetExceeded = false;
   let canceled = false;
   let unchanged = 0;
-  let wrapUpLeft = Infinity;
   const maxTurns = maxActions * 3 + 20;
 
   // Notes waiting for the next observation, which is the screen they are
@@ -388,7 +398,7 @@ async function playLoop({
   latestObservation = first.text;
   visualObservation = Boolean(first.screenshotBase64);
   transcript.write({ kind: 'observation', step: first.step, text: first.text, url: first.url });
-  const landing = await safeScreenshot(driver, 'landing', screenshots, transcript);
+  const landing = await safeScreenshot(driver, 'landing', screenshots, transcript, observations.length);
   messages.push({
     role: 'user',
     content: [
@@ -406,13 +416,9 @@ async function playLoop({
       quitDetail = 'the job was cancelled';
       break;
     }
-    if (wrapUpLeft <= 0) {
-      quitReason = 'budget_exhausted';
-      break;
-    }
-
     turns += 1;
     let answer;
+    const askedAt = Date.now();
     try {
       answer = await callWithImageRetry({
         client,
@@ -445,30 +451,28 @@ async function playLoop({
       throw error;
     }
 
-    ledger.record({ model: answer.model, usage: answer.usage });
+    ledger.record({ model: answer.model, usage: answer.usage, ms: Date.now() - askedAt });
     // The warn turn is a turn the developer paid for, so its tool calls are
     // executed like any other and the wrap-up notice rides along on that
     // turn's tool_result. Two things this avoids: discarding a paid action,
     // and pushing a second consecutive `user` message, which the Messages API
     // refuses when the previous turn ended in a tool_use.
-    let wrapUpNotice = null;
-    if (answer.budget === 'warn' && !budgetWarned) {
+    const warnedNow = answer.budget === 'warn' && !budgetWarned;
+    if (warnedNow) {
       budgetWarned = true;
-      wrapUpLeft = WRAP_UP_TURNS;
       // Surfaced, not swallowed: this is the developer's money and the one
       // moment where telling them costs nothing and saying nothing costs a
-      // report.
-      log(`budget warning: this run has used at least 80 percent of its model budget. Wrapping up after ${WRAP_UP_TURNS} more turns so the report still gets written.`);
-      wrapUpNotice = `You are nearly out of budget for this session. Take a final screenshot if you need one, then stop calling tools so the report can be written. You have ${WRAP_UP_TURNS} turns left.`;
+      // report. Recorded before anything else can fail, so the journal and
+      // the transcript both say the run was warned.
+      log('budget warning: this run has used at least 80 percent of its model budget. Playing this turn\'s actions, then stopping so the rest of the budget goes to the report.');
       transcript.write({ kind: 'budget_warning', wrap_up_turns: WRAP_UP_TURNS });
+      if (typeof onBudgetWarning === 'function') await onBudgetWarning();
     }
-    if (budgetWarned) wrapUpLeft -= 1;
 
     if (answer.toolUses.length === 0) {
       // No tool call is how a persona stops: there is no quit tool in the
-      // step's allowlist, and adding one would be refused by the proxy. A warn
-      // on such a turn needs no notice either, since the loop ends here.
-      quitReason = budgetWarned && wrapUpNotice ? 'budget_exhausted' : 'persona_quit';
+      // step's allowlist, and adding one would be refused by the proxy.
+      quitReason = warnedNow ? 'budget_exhausted' : 'persona_quit';
       quitDetail = answer.text.slice(0, 400);
       transcript.write({ kind: 'quit', reason: quitReason, text: answer.text });
       break;
@@ -485,15 +489,25 @@ async function playLoop({
       }
       transcript.write({ kind: 'action_selection', source: selection.source || 'jev', status: selection.status, confidence: selection.confidence, selected_tool_id: selection.toolId, reason: selection.reason });
       if (selection.status !== 'selected') { quitReason = 'persona_quit'; quitDetail = 'Ravensight Playtester stopped for review: ' + selection.reason; break; }
-      if (selection.budget === 'warn') { budgetWarned = true; wrapUpLeft = Math.min(wrapUpLeft, WRAP_UP_TURNS); }
+      if (selection.budget === 'warn' && !budgetWarned) {
+        budgetWarned = true;
+        log('budget warning: this run has used at least 80 percent of its model budget. Playing this turn\'s actions, then stopping so the rest of the budget goes to the report.');
+        transcript.write({ kind: 'budget_warning', wrap_up_turns: WRAP_UP_TURNS });
+        if (typeof onBudgetWarning === 'function') await onBudgetWarning();
+      }
     }
     messages.push({ role: 'assistant', content: assistantContent(answer) });
     const results = [];
+    // Every ref in this turn was read off the screen before it. Once one of
+    // its actions changes the screen, a later ref may name something else.
+    let screenChangedThisTurn = false;
     for (const toolUse of answer.toolUses) {
       if (selection && toolUse.id !== selection.toolId) { results.push({ type: 'tool_result', tool_use_id: toolUse.id, content: 'Alternative not selected. No action executed.' }); continue; }
+      const notesBefore = notes.length;
       const executed = await executeTool({
         driver,
         toolUse,
+        staleRefs: screenChangedThisTurn,
         onNote: note => {
           // The note is nested rather than spread: its own `kind`
           // (bug, confusing, good, suggestion, reaction) would otherwise
@@ -503,12 +517,13 @@ async function playLoop({
           transcript.write({ kind: 'note', tool: toolUse.name, note });
         },
         onScreenshot: shot => {
-          screenshots.push({ ...shot, order: nextOrder() });
+          screenshots.push({ ...shot, observation: observations.length, order: nextOrder() });
           transcript.write({ kind: 'screenshot', path: shot.relativePath, reason: shot.reason });
         }
       });
       results.push(executed.result);
       if (executed.spentAction) actions += 1;
+      if (executed.changed && ACTION_TOOLS.includes(toolUse.name)) screenChangedThisTurn = true;
       transcript.write({
         kind: 'action',
         tool: toolUse.name,
@@ -519,8 +534,18 @@ async function playLoop({
         changed: executed.changed,
         actions
       });
+      if (executed.driverError) {
+        // The driver failed, not the game: a target that matched twice,
+        // locator code, a ref not on screen, a timeout on an unchanged page.
+        // Recorded as its own kind, and the observation and any note written
+        // on this call are marked, so the report step can be told (and the
+        // validator can check) that none of it is evidence of a game bug.
+        for (const note of notes.slice(notesBefore)) note.driver_error = true;
+        transcript.write({ kind: 'driver_error', tool: toolUse.name, error: executed.error, observation: observations.length + (executed.observation ? 1 : 0) });
+      }
       if (executed.observation) {
-        observed(executed.observation);
+        const entry = observed(executed.observation);
+        if (executed.driverError) entry.driver_error = String(executed.error || 'driver error').slice(0, 200);
         latestObservation = executed.observation.text;
         visualObservation = Boolean(executed.observation.screenshotBase64);
         recentActions.push(`${toolUse.name}: ${executed.changed ? 'screen changed' : 'no observed change'}`);
@@ -552,7 +577,7 @@ async function playLoop({
       // with the persona's view of the screen in front of it. A 4xx on a
       // resource is a `network` note, which the report step reads as context
       // rather than as a bug (a missing favicon is not a broken game).
-      const shot = await safeScreenshot(driver, 'console-error', screenshots, transcript);
+      const shot = await safeScreenshot(driver, 'console-error', screenshots, transcript, observations.length);
       for (const entry of consoleErrors) {
         const kind = RESOURCE_4XX.test(String(entry.text)) ? 'network' : 'bug';
         const note = { kind, text: `${entry.type}: ${entry.text}`, automatic: true, ...(shot ? { screenshot: shot.relativePath } : {}) };
@@ -562,8 +587,6 @@ async function playLoop({
       asides.push(`The game logged ${consoleErrors.length} error(s): ${consoleErrors.map(entry => entry.text).join(' | ').slice(0, 800)}${shot ? ` A screenshot is saved at ${shot.relativePath}.` : ''}`);
     }
 
-    if (wrapUpNotice) asides.push(wrapUpNotice);
-
     const wedged = unchanged >= STUCK_LIMIT;
     if (!wedged && unchanged > 0 && unchanged % STUCK_HINT_AFTER === 0) {
       transcript.write({ kind: 'stuck', unchanged, fatal: false });
@@ -585,6 +608,12 @@ async function playLoop({
     }
 
     if (heartbeat) await heartbeat.beat({ actionsTaken: actions, checkpointStep: turns });
+
+    if (budgetWarned) {
+      quitReason = 'budget_exhausted';
+      quitDetail = 'stopped at the 80 percent budget warning so the report could be written';
+      break;
+    }
   }
 
   if (actions >= maxActions && quitReason === 'budget_exhausted') {
@@ -592,7 +621,7 @@ async function playLoop({
     quitDetail = `the action budget of ${maxActions} ran out`;
   }
 
-  const final = await safeScreenshot(driver, 'final', screenshots, transcript);
+  const final = await safeScreenshot(driver, 'final', screenshots, transcript, observations.length);
   return {
     actionsTaken: actions,
     turns,
@@ -646,17 +675,22 @@ async function reportPhase({
         notes.length > 0 ? notes.map((note, i) => noteLine(note, i)).join('\n') : '(none)',
         '',
         'Notes marked automatic were recorded by the runner from the console, with no severity: decide one from what the persona saw. Network notes are context, not bugs: a failed request is a finding only when the persona saw a broken screen because of it. A finding whose only evidence is automatic network notes is capped at minor by the runner.',
+        'Driver errors are never evidence of a game bug. An observation marked [driver error: ...] or a note "written on a driver error" means the runner could not carry out the action (a target that matched two elements, locator code, a ref not on screen, a timeout on a page that did not change); the game did nothing wrong there. Never report a blocked loop, a frozen screen or a broken button from them. The runner drops any blocker or major whose only cited observations are driver errors.',
+        'Notes of kind waiting ("waiting for X to finish") are context, never a finding of any severity, and the runner drops a finding cited only by them. Before reporting a loading problem, check the later observations and screenshots: if they show the content loaded, it is not a finding.',
         '',
         'Observations the persona was shown, by index:',
         observationLines(observations),
         '',
         'Each note carries "after": the URL and title of the observation that followed it. A blocker or major finding must cite the observation index where the failure is visible, as evidence {"type": "transcript_step", "ref": "observation N"}, and must not be contradicted by its note\'s "after" state or by a later persona note: if the next screen or a later note shows the game working, it is not a blocker or a major. Cite a note as {"type": "transcript_step", "ref": "note N"}.',
+        'A blocker or major finding must also cite the screenshot taken at or after that observation (the list below says which observation each one followed), and that screenshot must not contradict it: if it shows the thing the finding says is missing or broken, such as the field or the button, it is not a blocker or a major. A text observation can be cut short on a long page; the screenshot is what the player saw.',
+        '',
+        'Every evidence entry is an object with exactly two keys, "type" and "ref", and nothing else. A screenshot is {"type": "screenshot", "ref": "screenshots/04-example.png"}, never "path" or "file".',
         '',
         'Never write calendar dates or times in the prose of report.md or report.json. The runner stamps when the report was generated, so leave the template\'s Date out rather than writing a placeholder for it.',
         'Replace every <...> slot in the template with real content. Never copy a <...> placeholder into the report.',
         '',
         'Screenshots available as evidence (use these exact paths in evidence refs):',
-        screenshots.length > 0 ? screenshots.map(shot => `- ${shot.relativePath}${shot.reason ? ` (${shot.reason})` : ''}`).join('\n') : '(none)',
+        screenshots.length > 0 ? screenshots.map(screenshotLine).join('\n') : '(none)',
         '',
         `report.md must contain these section headings, spelled exactly: ${REQUIRED_MARKDOWN_SECTIONS.map(section => `"${section}"`).join(', ')}.`,
         'Write report.md first, then report.json. Leave usage as zeros; the runner fills it in.'
@@ -667,24 +701,70 @@ async function reportPhase({
   let report = null;
   let markdown = null;
   let lint = [];
-
-  // Two attempts at most: the first, and one repair if the schema or the
-  // evidence refs refuse it. Each attempt is a metered question, and the
-  // dogfood gate asks for "first or the repair attempt", not an
-  // open ended loop.
-  for (let attempt = 1; attempt <= 2; attempt += 1) {
-    const answer = await callStepWithRetry({
-      client,
-      jobId,
-      runId,
-      step: STEPS.personaReport,
-      system,
-      messages,
-      tools: REPORT_TOOLS,
-      maxTokens: REPORT_MAX_TOKENS,
-      escalate: shouldEscalate(persona)
-    });
-    ledger.record({ model: answer.model, usage: answer.usage });
+  // The newest report.json the model wrote, across turns: a repair turn that
+  // rewrites only report.md keeps the report.json it already wrote.
+  let jsonText = null;
+  let repaired = false;
+  let continued = false;
+  let droppedByRunner = [];
+
+  // One repair at most when the schema or the evidence refs refuse what was
+  // written: each turn is a metered question, and the dogfood gate asks for
+  // "first or the repair attempt", not an open ended loop. A turn that wrote
+  // report.md and stopped before report.json is not a refused report, so it
+  // gets one plain "now write report.json" that does not spend the repair.
+  // In 0.2.8 it did, and the report.json that came back had its five
+  // screenshot refs under "path" with no turn left to fix them.
+  // A warned run has at most a fifth of its budget left, and the proxy
+  // reserves each call's worst case before sending it. So the report step
+  // never escalates to a dearer tier after a warning, and a 402 on a report
+  // turn gets one retry with the output ceiling cut to what the refusal says
+  // still fits.
+  const escalate = shouldEscalate(persona) && !play.budgetWarned;
+  let maxTokens = REPORT_MAX_TOKENS;
+  let shrunk = false;
+  let lastCandidate = null;
+  for (let attempt = 1; ; attempt += 1) {
+    let answer;
+    const askedAt = Date.now();
+    try {
+      answer = await callStepWithRetry({
+        client,
+        jobId,
+        runId,
+        step: STEPS.personaReport,
+        system,
+        messages,
+        tools: REPORT_TOOLS,
+        maxTokens,
+        escalate
+      });
+    } catch (error) {
+      if (!(error instanceof ProxyError && error.isBudget)) throw error;
+      const bodyChars = JSON.stringify({ messages, system, tools: REPORT_TOOLS, max_tokens: maxTokens, stream: true }).length;
+      const smaller = shrunk ? null : reportTokensThatFit(error, maxTokens, bodyChars);
+      if (!smaller && !shrunk && !lastCandidate) {
+        log(hasBudgetFigures(error)
+          ? `even the smallest report (${REPORT_MIN_TOKENS} output tokens) does not fit the remaining model budget once the request itself is paid for, so the report is written from the notes instead.`
+          : 'the report turn was refused for budget with no budget figures in the refusal, so there is nothing to size a retry from; the report is written from the notes instead.');
+      }
+      if (smaller) {
+        shrunk = true;
+        transcript.write({ kind: 'report_budget_retry', attempt, max_tokens: smaller, figures: error.figures || {} });
+        log(`the report turn did not fit the remaining model budget; asking again with an output ceiling of ${smaller} tokens.`);
+        maxTokens = smaller;
+        continue;
+      }
+      // A report already written on an earlier turn is the report, with its
+      // lint, rather than a partial one rebuilt from the notes.
+      if (lastCandidate) {
+        report = lastCandidate;
+        log(`the model budget ran out on a follow-up report turn; keeping the report.json already written (${lint.length} problem(s)).`);
+        break;
+      }
+      throw error;
+    }
+    ledger.record({ model: answer.model, usage: answer.usage, ms: Date.now() - askedAt });
     transcript.write({ kind: 'report_turn', attempt, stop_reason: answer.stopReason, files: answer.toolUses.map(t => t.input && t.input.path) });
 
     const files = new Map();
@@ -695,12 +775,13 @@ async function reportPhase({
       files.set(name, String(toolUse.input.content || ''));
     }
     if (files.has(RUN_FILES.reportMd)) markdown = files.get(RUN_FILES.reportMd);
+    if (files.has(RUN_FILES.reportJson)) jsonText = files.get(RUN_FILES.reportJson);
 
     let parsed = null;
     let parseError = null;
-    if (files.has(RUN_FILES.reportJson)) {
+    if (jsonText !== null) {
       try {
-        parsed = JSON.parse(files.get(RUN_FILES.reportJson));
+        parsed = JSON.parse(jsonText);
       } catch (error) {
         parseError = String(error && error.message);
       }
@@ -720,7 +801,20 @@ async function reportPhase({
       quitReason: play.quitReason,
       quitDetail: play.quitDetail
     };
-    const candidate = parsed ? capNetworkOnlyFindings(stampReport(parsed, facts), { notes, screenshots }) : null;
+    // An evidence path under the wrong key is moved to `ref` before anything
+    // judges the report; see normaliseEvidence.
+    const normalised = parsed ? normaliseEvidence(parsed, runDirectory) : null;
+    if (normalised && normalised.changed > 0) {
+      transcript.write({ kind: 'report_evidence_normalised', attempt, entries: normalised.changed });
+    }
+    const capped = normalised ? capNetworkOnlyFindings(stampReport(normalised.report, facts), { notes, screenshots }) : null;
+    const runnerDrop = capped ? dropRunnerRejectedFindings(capped, { notes, observations }) : { report: null, dropped: [] };
+    const candidate = runnerDrop.report;
+    for (const entry of runnerDrop.dropped) {
+      transcript.write({ kind: 'finding_dropped', id: entry.finding.id, severity: entry.finding.severity, title: entry.finding.title, reason: entry.reason });
+    }
+    droppedByRunner = runnerDrop.dropped;
+    if (candidate) lastCandidate = candidate;
     const validation = candidate ? validator.validate(candidate) : { ok: false, details: [{ instancePath: '/', message: parseError || 'report.json was not written' }] };
     const evidenceProblems = candidate ? await checkEvidence(candidate, runDirectory) : [];
     // The server lints the same four sections at `complete` and records the
@@ -742,30 +836,49 @@ async function reportPhase({
       ...evidenceProblems,
       ...missingSections.map(section => `report.md has no "${section}" section`)
     ];
-    if (attempt === 2) {
+
+    // Every tool_use the next request echoes back needs its tool_result, or
+    // the API refuses the request. A write cut off at the token limit is
+    // malformed: it was not saved, and it is answered so.
+    const toolResults = answer.toolUses.map(toolUse => (toolUse.malformed
+      ? { type: 'tool_result', tool_use_id: toolUse.id, is_error: true, content: [{ type: 'text', text: CUT_OFF_WRITE }] }
+      : { type: 'tool_result', tool_use_id: toolUse.id, content: [{ type: 'text', text: 'saved' }] }));
+
+    if (jsonText === null && files.has(RUN_FILES.reportMd) && !continued) {
+      continued = true;
+      messages.push({ role: 'assistant', content: assistantContent(answer) });
+      messages.push({ role: 'user', content: toolResults.concat([{ type: 'text', text: CONTINUE_WITH_JSON }]) });
+      continue;
+    }
+
+    if (repaired) {
       if (!candidate) throw new Error(`the report step did not produce a valid report.json: ${lint.join('; ')}`);
-      // Second attempt still imperfect: keep it, record the lint, and let
+      // Still imperfect after the repair: keep it, record the lint, and let
       // the server have the final word. It validates the same schema and
       // drops unevidenced findings itself, so this is a report with known
-      // problems rather than no report at all.
+      // problems rather than no report at all. Every problem on its own line,
+      // before anything is uploaded, so a refusal at complete is never the
+      // first the developer hears of it.
       report = candidate;
-      log(`report.json still has problems after one repair: ${lint.join('; ')}`);
+      log(`report.json still has problems after one repair (${lint.length}); the server will refuse a report that fails its schema:\n${lint.map(line => `  - ${line}`).join('\n')}`);
       break;
     }
 
+    repaired = true;
     messages.push({ role: 'assistant', content: assistantContent(answer) });
     messages.push({
       role: 'user',
-      content: answer.toolUses
-        .filter(toolUse => !toolUse.malformed)
-        .map(toolUse => ({ type: 'tool_result', tool_use_id: toolUse.id, content: [{ type: 'text', text: 'saved' }] }))
-        .concat([{ type: 'text', text: repairInstruction(validation.details, evidenceProblems, missingSections) }])
+      content: toolResults.concat([{ type: 'text', text: repairInstruction(validation.details, evidenceProblems, missingSections) }])
     });
   }
 
   if (!markdown) {
     markdown = renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason: 'The report step wrote report.json but no report.md.' });
   }
+  if (droppedByRunner.length > 0) {
+    markdown = rewriteMarkdown(markdown, droppedByRunner.map(entry => entry.finding), report.findings || [], RUNNER_DROPPED_SECTION);
+    log(`dropped ${droppedByRunner.length} finding(s) the runner's evidence rules refuse: ${droppedByRunner.map(entry => `${entry.finding.id} (${entry.reason})`).join(', ')}`);
+  }
   const generatedAt = new Date().toISOString();
   const facts = reportFacts({ persona, driverName, target, play });
   markdown = stampGeneratedAt(markdown, generatedAt, facts);
@@ -776,6 +889,72 @@ async function reportPhase({
   return { report, markdown, lint };
 }
 
+/** The smallest output ceiling worth retrying a report turn with. */
+export const REPORT_MIN_TOKENS = 1536;
+
+/** Whether a 402 carried the three figures a retry is sized from. */
+export function hasBudgetFigures(error) {
+  const figures = (error && error.figures) || {};
+  return ['usd_used', 'usd_max', 'projected_usd'].every(key => Number.isFinite(Number(figures[key])) && figures[key] !== null && figures[key] !== '');
+}
+
+/** Characters per input token, for estimating the input share of a projection. */
+export const CHARS_PER_INPUT_TOKEN = 3.5;
+
+/**
+ * Output price per token over input price per token, assumed when the 402
+ * does not split its projection. Prices are server side only; this ratio is
+ * only used to divide the server's own projected figure into two parts.
+ */
+export const ASSUMED_OUTPUT_PRICE_RATIO = 5;
+
+/**
+ * The output ceiling a refused report turn could fit in, or null when even
+ * the minimum report cannot fit.
+ *
+ * The proxy's projection prices the input and the output ceiling together,
+ * and only the output ceiling can move, so the input's share is taken out of
+ * the room first: from the 402's own split (`projected: { input_cents,
+ * output_cents }`) when the server sends one, otherwise estimated from the
+ * request's size (characters / 3.5 at an input price a fifth of the output
+ * price). What is left of the room is then shared out over the output, with
+ * a tenth of headroom.
+ *
+ * @param {ProxyError} error
+ * @param {number} current the output ceiling that was refused
+ * @param {number} [bodyChars] the refused request's JSON size
+ * @returns {number|null}
+ */
+export function reportTokensThatFit(error, current, bodyChars = 0) {
+  const figures = (error && error.figures) || {};
+  const used = Number(figures.usd_used);
+  const max = Number(figures.usd_max);
+  const projected = Number(figures.projected_usd);
+  if (![used, max, projected].every(Number.isFinite) || projected <= 0 || current <= 0) return null;
+  const room = max - used;
+  if (room <= 0 || room >= projected) return null;
+  const split = figures.projected && typeof figures.projected === 'object' ? figures.projected : null;
+  let inputUsd;
+  let outputUsd;
+  if (split && Number.isFinite(Number(split.input_cents)) && Number.isFinite(Number(split.output_cents))) {
+    inputUsd = Number(split.input_cents) / 100;
+    outputUsd = Number(split.output_cents) / 100;
+  } else {
+    // The request's JSON size counts any base64 screenshot as text, which
+    // overstates the input (an image is billed far below its base64 length
+    // as text). Overstating the input understates what the output can have,
+    // so the estimate errs on the safe side: a retry it allows should fit.
+    const inputTokens = Math.max(0, Number(bodyChars) || 0) / CHARS_PER_INPUT_TOKEN;
+    const perInputToken = projected / (inputTokens + ASSUMED_OUTPUT_PRICE_RATIO * current);
+    inputUsd = perInputToken * inputTokens;
+    outputUsd = projected - inputUsd;
+  }
+  const outputRoom = room - inputUsd;
+  if (outputRoom <= 0 || outputUsd <= 0) return null;
+  const tokens = Math.floor(current * (outputRoom / outputUsd) * 0.9);
+  return tokens >= REPORT_MIN_TOKENS ? tokens : null;
+}
+
 /**
  * `callStepWithRetry`, plus the one retry that is about our own request
  * rather than the server's mood: `too_many_images` means this step allows
@@ -901,10 +1080,10 @@ function redactToolInput(toolUse) {
   return rest;
 }
 
-async function safeScreenshot(driver, name, screenshots, transcript) {
+async function safeScreenshot(driver, name, screenshots, transcript, observation = null) {
   try {
     const shot = await driver.screenshot(name);
-    screenshots.push({ ...shot, reason: name, order: nextOrder() });
+    screenshots.push({ ...shot, reason: name, ...(Number.isInteger(observation) ? { observation } : {}), order: nextOrder() });
     transcript.write({ kind: 'screenshot', path: shot.relativePath, reason: name });
     return shot;
   } catch (error) {
@@ -948,6 +1127,7 @@ async function finish(outcome, ledger, runDirectory, transcript, heartbeat) {
     }
   }
   outcome.usage = ledger.toReportUsage();
+  outcome.duration = ledger.duration();
   await writeUsageFile(runDirectory, ledger);
   transcript.write({ kind: 'end', state: outcome.state, quit_reason: outcome.quitReason, actions: outcome.actionsTaken });
   await transcript.close();
@@ -1036,6 +1216,8 @@ export function notesToFindings(notes, screenshots) {
   for (const note of notes) {
     const category = NOTE_CATEGORY[note.kind];
     if (!category || !note.text) continue;
+    // A blocker or major written on a driver error is the runner's failure.
+    if (note.driver_error && ABOVE_MINOR.includes(note.severity)) continue;
     const text = String(note.text);
     const shot = nearestScreenshot(note, screenshots);
     findings.push({
@@ -1113,6 +1295,7 @@ async function writePartialMarkdown({ runDirectory, persona, driverName, target,
 export function noteLine(note, i) {
   const tags = [
     note.automatic ? 'automatic' : null,
+    note.driver_error ? 'written on a driver error' : null,
     Number.isInteger(note.observation) && note.observation > 0 ? `seen at observation ${note.observation}` : null,
     note.screenshot ? `screenshot ${note.screenshot}` : null
   ].filter(Boolean);
@@ -1122,13 +1305,35 @@ export function noteLine(note, i) {
   return `${i + 1}. [${note.kind}${note.severity ? `/${note.severity}` : ''}] ${note.text}${tags.length > 0 ? ` (${tags.join(', ')})` : ''}${after}`;
 }
 
+/** The tool_result for a write_file call that was cut off before its input was complete. */
+export const CUT_OFF_WRITE = 'That write_file call was cut off before it finished, so nothing was saved. Write that file again, shorter if it has to be.';
+
+/** The one follow-up a report turn gets for writing report.md and stopping. */
+export const CONTINUE_WITH_JSON = 'report.md is saved. Now write report.json with write_file. Every evidence entry is exactly {"type": ..., "ref": ...}; a screenshot is {"type": "screenshot", "ref": "screenshots/04-example.png"}.';
+
+/**
+ * A screenshot as one line of the report prompt: its path, the observation
+ * it followed (what a blocker or major has to be checked against) and why
+ * it was taken.
+ *
+ * @param {Object} shot
+ * @returns {string}
+ */
+export function screenshotLine(shot) {
+  const tags = [
+    Number.isInteger(shot.observation) && shot.observation > 0 ? `after observation ${shot.observation}` : null,
+    shot.reason ? String(shot.reason) : null
+  ].filter(Boolean);
+  return `- ${shot.relativePath}${tags.length > 0 ? ` (${tags.join('; ')})` : ''}`;
+}
+
 /** The most observations listed in the report prompt; the newest are kept. */
 export const MAX_OBSERVATIONS_IN_REPORT = 200;
 
 function observationLines(observations) {
   if (!observations || observations.length === 0) return '(none)';
   const shown = observations.slice(-MAX_OBSERVATIONS_IN_REPORT);
-  const lines = shown.map(entry => `observation ${entry.index}: step ${entry.step}${entry.url ? ` ${entry.url}` : ''}${entry.title ? ` "${entry.title}"` : ''}`);
+  const lines = shown.map(entry => `observation ${entry.index}: step ${entry.step}${entry.url ? ` ${entry.url}` : ''}${entry.title ? ` "${entry.title}"` : ''}${entry.driver_error ? ` [driver error: ${entry.driver_error}]` : ''}`);
   if (shown.length < observations.length) lines.unshift(`(the first ${observations.length - shown.length} are left out)`);
   return lines.join('\n');
 }
@@ -1161,6 +1366,65 @@ export function observationTitle(observation) {
  * @param {{notes: Array<Object>, screenshots: Array<Object>}} context
  * @returns {Object} the report, with capped findings rewritten
  */
+/** How report.md lists what the runner dropped. */
+export const RUNNER_DROPPED_SECTION = Object.freeze({
+  heading: '## Dropped by the runner (driver errors or waiting notes only)',
+  explanation: 'The runner removes a blocker or major whose only cited observations are driver errors (the runner could not carry out an action; the game did nothing wrong), and any finding cited only by waiting notes. They are not sent to the server. They are kept here so you can judge them yourself.',
+  emptyNote: '- Nothing here that the runner kept. See the dropped findings below.'
+});
+
+/**
+ * The findings the runner refuses before anything is uploaded, and the
+ * report without them.
+ *
+ * - A blocker or major whose transcript evidence (observations and notes it
+ *   cites) all points at driver errors. Screenshots do not rescue it: the
+ *   Texas run filed "the season loop is fully blocked" from three strict mode
+ *   failures, with a screenshot of the screen working.
+ * - Any finding whose transcript evidence is only waiting notes.
+ *
+ * A finding with no transcript evidence is left to the other checks.
+ *
+ * @param {Object} report
+ * @param {{notes?: Array, observations?: Array}} context
+ * @returns {{report: Object, dropped: Array<{finding: Object, reason: string}>}}
+ */
+export function dropRunnerRejectedFindings(report, { notes = [], observations = [] } = {}) {
+  if (!report || !Array.isArray(report.findings)) return { report, dropped: [] };
+  const byIndex = new Map(observations.map(entry => [entry.index, entry]));
+  const verdict = finding => {
+    const cited = [];
+    for (const item of Array.isArray(finding.evidence) ? finding.evidence : []) {
+      if (!item || item.type !== 'transcript_step') continue;
+      const ref = String(item.ref || '');
+      const observation = ref.match(/^\s*observation\s*#?\s*(\d+)\s*$/i);
+      const note = ref.match(/^\s*note\s*#?\s*(\d+)\s*$/i);
+      if (observation) {
+        const entry = byIndex.get(Number(observation[1]));
+        cited.push(entry && entry.driver_error ? 'driver' : 'other');
+      } else if (note) {
+        const entry = notes[Number(note[1]) - 1];
+        cited.push(entry && entry.kind === 'waiting' ? 'waiting' : entry && entry.driver_error ? 'driver' : 'other');
+      } else {
+        cited.push('other');
+      }
+    }
+    if (cited.length === 0) return null;
+    if (ABOVE_MINOR.includes(finding.severity) && cited.every(kind => kind === 'driver')) return 'driver errors only';
+    if (cited.every(kind => kind === 'waiting')) return 'waiting notes only';
+    if (ABOVE_MINOR.includes(finding.severity) && cited.every(kind => kind === 'driver' || kind === 'waiting')) return 'driver errors and waiting notes only';
+    return null;
+  };
+  const dropped = [];
+  const findings = [];
+  for (const finding of report.findings) {
+    const reason = finding ? verdict(finding) : null;
+    if (reason) dropped.push({ finding, reason });
+    else findings.push(finding);
+  }
+  return { report: dropped.length > 0 ? { ...report, findings } : report, dropped };
+}
+
 export function capNetworkOnlyFindings(report, { notes = [], screenshots = [] } = {}) {
   if (!report || !Array.isArray(report.findings)) return report;
   const byShot = new Map();