ravensight-playtest 0.2.9 → 0.3.1

package/src/run/personaLoop.js

@@ -1,3 +1,4 @@
+import { rewriteMarkdown } from './dropped.js';
 import { selectAction, JEV_CONTROLLER_PROMPT } from './actionSelector.js';
 import { mkdir, writeFile } from 'node:fs/promises';
 import path from 'node:path';
@@ -69,6 +70,8 @@ const ABOVE_MINOR = Object.freeze(['blocker', 'major']);
 export const NETWORK_CAP_NOTE = 'Severity capped at minor by the runner: the only evidence is automatic network notes, and a failed request is a finding above minor only when the persona saw the game break because of it.';
 
 /** A persona note kind as a report finding category. */
+// `waiting` has no category on purpose: a "waiting for X to finish" note is
+// context, and nothing turns it into a finding.
 export const NOTE_CATEGORY = Object.freeze({ bug: 'bug', confusing: 'ux', suggestion: 'ux', good: 'ux', reaction: 'ux' });
 
 /** Stop injecting the "nothing changed" hint after this many identical screens. */
@@ -77,8 +80,16 @@ export const STUCK_HINT_AFTER = 5;
 /** Give up entirely after this many identical screens: the persona is wedged. */
 export const STUCK_LIMIT = 12;
 
-/** Turns allowed after a budget warning, to land the report. */
-export const WRAP_UP_TURNS = 3;
+/**
+ * Act turns allowed after a budget warning: none. The proxy reserves each
+ * call's pessimistic projection (the whole request at the full input rate,
+ * plus the output ceiling) before it sends it, so every act turn after the
+ * warning ate into the 20 percent the report step needs, and in the Texas run
+ * three of them left too little for the report's own reservation: the report
+ * call was refused with 402 and the run ended at cost_cap with no report.
+ * The warned turn's own actions are still played; then the report is written.
+ */
+export const WRAP_UP_TURNS = 0;
 
 /** Token ceiling asked for per act turn. The proxy clamps it to the step's own. */
 export const ACT_MAX_TOKENS = 1024;
@@ -330,9 +341,11 @@ async function playLoop({
   keepObservations,
   heartbeat,
   onProgress = () => {},
+  onBudgetWarning = null,
   log = () => {}
 }) {
-  const system = buildSystem({ skill, persona, brief, intentNotes, driverName, target, maxActions });
+  const system = buildSystem({ skill, persona, brief, intentNotes, driverName, target, maxActions, runId });
+  const identity = runIdentity({ persona, runId });
   if (controller === 'jev') system.push({ type: 'text', text: JEV_CONTROLLER_PROMPT });
   const availableTools = driver.supportedActions
     ? [...PLAY_TOOLS, ...(driver.supportedActions.includes('swipe') ? [SWIPE_TOOL] : [])].filter(tool => !ACTION_TOOLS.includes(tool.name) || driver.supportedActions.includes(tool.name))
@@ -350,6 +363,9 @@ async function playLoop({
     }
   })) : availableTools;
   const recentActions = [];
+  // The observation and action lines of the transcript, kept for the report
+  // step's "Actions you took" list.
+  const actionRecords = [];
   let latestObservation = '';
   let visualObservation = false;
   const messages = [];
@@ -361,7 +377,6 @@ async function playLoop({
   let budgetExceeded = false;
   let canceled = false;
   let unchanged = 0;
-  let wrapUpLeft = Infinity;
   const maxTurns = maxActions * 3 + 20;
 
   // Notes waiting for the next observation, which is the screen they are
@@ -387,6 +402,7 @@ async function playLoop({
   latestObservation = first.text;
   visualObservation = Boolean(first.screenshotBase64);
   transcript.write({ kind: 'observation', step: first.step, text: first.text, url: first.url });
+  actionRecords.push({ kind: 'observation', text: first.text });
   const landing = await safeScreenshot(driver, 'landing', screenshots, transcript, observations.length);
   messages.push({
     role: 'user',
@@ -405,13 +421,9 @@ async function playLoop({
       quitDetail = 'the job was cancelled';
       break;
     }
-    if (wrapUpLeft <= 0) {
-      quitReason = 'budget_exhausted';
-      break;
-    }
-
     turns += 1;
     let answer;
+    const askedAt = Date.now();
     try {
       answer = await callWithImageRetry({
         client,
@@ -444,30 +456,28 @@ async function playLoop({
       throw error;
     }
 
-    ledger.record({ model: answer.model, usage: answer.usage });
+    ledger.record({ model: answer.model, usage: answer.usage, ms: Date.now() - askedAt });
     // The warn turn is a turn the developer paid for, so its tool calls are
     // executed like any other and the wrap-up notice rides along on that
     // turn's tool_result. Two things this avoids: discarding a paid action,
     // and pushing a second consecutive `user` message, which the Messages API
     // refuses when the previous turn ended in a tool_use.
-    let wrapUpNotice = null;
-    if (answer.budget === 'warn' && !budgetWarned) {
+    const warnedNow = answer.budget === 'warn' && !budgetWarned;
+    if (warnedNow) {
       budgetWarned = true;
-      wrapUpLeft = WRAP_UP_TURNS;
       // Surfaced, not swallowed: this is the developer's money and the one
       // moment where telling them costs nothing and saying nothing costs a
-      // report.
-      log(`budget warning: this run has used at least 80 percent of its model budget. Wrapping up after ${WRAP_UP_TURNS} more turns so the report still gets written.`);
-      wrapUpNotice = `You are nearly out of budget for this session. Take a final screenshot if you need one, then stop calling tools so the report can be written. You have ${WRAP_UP_TURNS} turns left.`;
+      // report. Recorded before anything else can fail, so the journal and
+      // the transcript both say the run was warned.
+      log('budget warning: this run has used at least 80 percent of its model budget. Playing this turn\'s actions, then stopping so the rest of the budget goes to the report.');
       transcript.write({ kind: 'budget_warning', wrap_up_turns: WRAP_UP_TURNS });
+      if (typeof onBudgetWarning === 'function') await onBudgetWarning();
     }
-    if (budgetWarned) wrapUpLeft -= 1;
 
     if (answer.toolUses.length === 0) {
       // No tool call is how a persona stops: there is no quit tool in the
-      // step's allowlist, and adding one would be refused by the proxy. A warn
-      // on such a turn needs no notice either, since the loop ends here.
-      quitReason = budgetWarned && wrapUpNotice ? 'budget_exhausted' : 'persona_quit';
+      // step's allowlist, and adding one would be refused by the proxy.
+      quitReason = warnedNow ? 'budget_exhausted' : 'persona_quit';
       quitDetail = answer.text.slice(0, 400);
       transcript.write({ kind: 'quit', reason: quitReason, text: answer.text });
       break;
@@ -484,15 +494,26 @@ async function playLoop({
       }
       transcript.write({ kind: 'action_selection', source: selection.source || 'jev', status: selection.status, confidence: selection.confidence, selected_tool_id: selection.toolId, reason: selection.reason });
       if (selection.status !== 'selected') { quitReason = 'persona_quit'; quitDetail = 'Ravensight Playtester stopped for review: ' + selection.reason; break; }
-      if (selection.budget === 'warn') { budgetWarned = true; wrapUpLeft = Math.min(wrapUpLeft, WRAP_UP_TURNS); }
+      if (selection.budget === 'warn' && !budgetWarned) {
+        budgetWarned = true;
+        log('budget warning: this run has used at least 80 percent of its model budget. Playing this turn\'s actions, then stopping so the rest of the budget goes to the report.');
+        transcript.write({ kind: 'budget_warning', wrap_up_turns: WRAP_UP_TURNS });
+        if (typeof onBudgetWarning === 'function') await onBudgetWarning();
+      }
     }
     messages.push({ role: 'assistant', content: assistantContent(answer) });
     const results = [];
+    // Every ref in this turn was read off the screen before it. Once one of
+    // its actions changes the screen, a later ref may name something else.
+    let screenChangedThisTurn = false;
     for (const toolUse of answer.toolUses) {
       if (selection && toolUse.id !== selection.toolId) { results.push({ type: 'tool_result', tool_use_id: toolUse.id, content: 'Alternative not selected. No action executed.' }); continue; }
+      const notesBefore = notes.length;
       const executed = await executeTool({
         driver,
         toolUse,
+        staleRefs: screenChangedThisTurn,
+        identity,
         onNote: note => {
           // The note is nested rather than spread: its own `kind`
           // (bug, confusing, good, suggestion, reaction) would otherwise
@@ -508,7 +529,8 @@ async function playLoop({
       });
       results.push(executed.result);
       if (executed.spentAction) actions += 1;
-      transcript.write({
+      if (executed.changed && ACTION_TOOLS.includes(toolUse.name)) screenChangedThisTurn = true;
+      const actionLine = {
         kind: 'action',
         tool: toolUse.name,
         input: redactToolInput(toolUse),
@@ -517,14 +539,27 @@ async function playLoop({
         error: executed.error,
         changed: executed.changed,
         actions
-      });
+      };
+      transcript.write(actionLine);
+      actionRecords.push(actionLine);
+      if (executed.driverError) {
+        // The driver failed, not the game: a target that matched twice,
+        // locator code, a ref not on screen, a timeout on an unchanged page.
+        // Recorded as its own kind, and the observation and any note written
+        // on this call are marked, so the report step can be told (and the
+        // validator can check) that none of it is evidence of a game bug.
+        for (const note of notes.slice(notesBefore)) note.driver_error = true;
+        transcript.write({ kind: 'driver_error', tool: toolUse.name, error: executed.error, observation: observations.length + (executed.observation ? 1 : 0) });
+      }
       if (executed.observation) {
-        observed(executed.observation);
+        const entry = observed(executed.observation);
+        if (executed.driverError) entry.driver_error = String(executed.error || 'driver error').slice(0, 200);
         latestObservation = executed.observation.text;
         visualObservation = Boolean(executed.observation.screenshotBase64);
         recentActions.push(`${toolUse.name}: ${executed.changed ? 'screen changed' : 'no observed change'}`);
         if (recentActions.length > 10) recentActions.shift();
         transcript.write({ kind: 'observation', step: executed.observation.step, text: executed.observation.text, url: executed.observation.url });
+        actionRecords.push({ kind: 'observation', text: executed.observation.text });
         // Only an input to the game can tell us anything about being stuck. A
         // `snapshot` or a `screenshot` never changes the screen by design, so
         // counting those would have a persona that looks twice before acting
@@ -554,15 +589,20 @@ async function playLoop({
       const shot = await safeScreenshot(driver, 'console-error', screenshots, transcript, observations.length);
       for (const entry of consoleErrors) {
         const kind = RESOURCE_4XX.test(String(entry.text)) ? 'network' : 'bug';
-        const note = { kind, text: `${entry.type}: ${entry.text}`, automatic: true, ...(shot ? { screenshot: shot.relativePath } : {}) };
+        const note = {
+          kind,
+          text: `${entry.type}: ${entry.text}`,
+          automatic: true,
+          ...(entry.url ? { url: String(entry.url).slice(0, 200) } : {}),
+          ...(entry.url && Number.isInteger(entry.status) ? { status: entry.status } : {}),
+          ...(shot ? { screenshot: shot.relativePath } : {})
+        };
         noted(note);
         transcript.write({ kind: 'note', note });
       }
       asides.push(`The game logged ${consoleErrors.length} error(s): ${consoleErrors.map(entry => entry.text).join(' | ').slice(0, 800)}${shot ? ` A screenshot is saved at ${shot.relativePath}.` : ''}`);
     }
 
-    if (wrapUpNotice) asides.push(wrapUpNotice);
-
     const wedged = unchanged >= STUCK_LIMIT;
     if (!wedged && unchanged > 0 && unchanged % STUCK_HINT_AFTER === 0) {
       transcript.write({ kind: 'stuck', unchanged, fatal: false });
@@ -584,6 +624,12 @@ async function playLoop({
     }
 
     if (heartbeat) await heartbeat.beat({ actionsTaken: actions, checkpointStep: turns });
+
+    if (budgetWarned) {
+      quitReason = 'budget_exhausted';
+      quitDetail = 'stopped at the 80 percent budget warning so the report could be written';
+      break;
+    }
   }
 
   if (actions >= maxActions && quitReason === 'budget_exhausted') {
@@ -601,7 +647,8 @@ async function playLoop({
     budgetExceeded,
     canceled,
     imageBudget,
-    finalScreenshot: final ? final.relativePath : null
+    finalScreenshot: final ? final.relativePath : null,
+    actionLog: actionLogFrom(actionRecords)
   };
 }
 
@@ -631,7 +678,7 @@ async function reportPhase({
   endedAt,
   log
 }) {
-  const system = buildSystem({ skill, persona, brief, intentNotes, driverName, target, maxActions: play.actionsTaken });
+  const system = buildSystem({ skill, persona, brief, intentNotes, driverName, target, maxActions: play.actionsTaken, runId });
   const messages = [{
     role: 'user',
     content: [{
@@ -645,10 +692,20 @@ async function reportPhase({
         notes.length > 0 ? notes.map((note, i) => noteLine(note, i)).join('\n') : '(none)',
         '',
         'Notes marked automatic were recorded by the runner from the console, with no severity: decide one from what the persona saw. Network notes are context, not bugs: a failed request is a finding only when the persona saw a broken screen because of it. A finding whose only evidence is automatic network notes is capped at minor by the runner.',
+        'Driver errors are never evidence of a game bug. An observation marked [driver error: ...] or a note "written on a driver error" means the runner could not carry out the action (a target that matched two elements, locator code, a ref not on screen, a timeout on a page that did not change); the game did nothing wrong there. Never report a blocked loop, a frozen screen or a broken button from them. The runner drops any blocker or major whose only cited observations are driver errors.',
+        'Notes of kind waiting ("waiting for X to finish") are context, never a finding of any severity, and the runner drops a finding cited only by them. Before reporting a loading problem, check the later observations and screenshots: if they show the content loaded, it is not a finding.',
         '',
         'Observations the persona was shown, by index:',
         observationLines(observations),
         '',
+        'Actions you took (every action that went through, numbered by action; "screen changed" means the screen after it differed from the screen before):',
+        actionLogLines(play.actionLog),
+        '',
+        'A claim in report.md or report.json that something was never done, never worked, did nothing or could not be done must agree with this list. If the list shows you clicked a control and the screen changed, do not write that you never clicked it or that it did nothing; describe what the screen showed after it instead. The runner downgrades a finding that says otherwise.',
+        '',
+        'Network notes (failed requests the runner recorded, with the URL and the status; list them under a "Network notes" heading in report.md when there are any):',
+        networkNoteLines(notes),
+        '',
         'Each note carries "after": the URL and title of the observation that followed it. A blocker or major finding must cite the observation index where the failure is visible, as evidence {"type": "transcript_step", "ref": "observation N"}, and must not be contradicted by its note\'s "after" state or by a later persona note: if the next screen or a later note shows the game working, it is not a blocker or a major. Cite a note as {"type": "transcript_step", "ref": "note N"}.',
         'A blocker or major finding must also cite the screenshot taken at or after that observation (the list below says which observation each one followed), and that screenshot must not contradict it: if it shows the thing the finding says is missing or broken, such as the field or the button, it is not a blocker or a major. A text observation can be cut short on a long page; the screenshot is what the player saw.',
         '',
@@ -674,6 +731,8 @@ async function reportPhase({
   let jsonText = null;
   let repaired = false;
   let continued = false;
+  let droppedByRunner = [];
+  let downgradedByRunner = [];
 
   // One repair at most when the schema or the evidence refs refuse what was
   // written: each turn is a metered question, and the dogfood gate asks for
@@ -682,19 +741,56 @@ async function reportPhase({
   // gets one plain "now write report.json" that does not spend the repair.
   // In 0.2.8 it did, and the report.json that came back had its five
   // screenshot refs under "path" with no turn left to fix them.
+  // A warned run has at most a fifth of its budget left, and the proxy
+  // reserves each call's worst case before sending it. So the report step
+  // never escalates to a dearer tier after a warning, and a 402 on a report
+  // turn gets one retry with the output ceiling cut to what the refusal says
+  // still fits.
+  const escalate = shouldEscalate(persona) && !play.budgetWarned;
+  let maxTokens = REPORT_MAX_TOKENS;
+  let shrunk = false;
+  let lastCandidate = null;
   for (let attempt = 1; ; attempt += 1) {
-    const answer = await callStepWithRetry({
-      client,
-      jobId,
-      runId,
-      step: STEPS.personaReport,
-      system,
-      messages,
-      tools: REPORT_TOOLS,
-      maxTokens: REPORT_MAX_TOKENS,
-      escalate: shouldEscalate(persona)
-    });
-    ledger.record({ model: answer.model, usage: answer.usage });
+    let answer;
+    const askedAt = Date.now();
+    try {
+      answer = await callStepWithRetry({
+        client,
+        jobId,
+        runId,
+        step: STEPS.personaReport,
+        system,
+        messages,
+        tools: REPORT_TOOLS,
+        maxTokens,
+        escalate
+      });
+    } catch (error) {
+      if (!(error instanceof ProxyError && error.isBudget)) throw error;
+      const bodyChars = JSON.stringify({ messages, system, tools: REPORT_TOOLS, max_tokens: maxTokens, stream: true }).length;
+      const smaller = shrunk ? null : reportTokensThatFit(error, maxTokens, bodyChars);
+      if (!smaller && !shrunk && !lastCandidate) {
+        log(hasBudgetFigures(error)
+          ? `even the smallest report (${REPORT_MIN_TOKENS} output tokens) does not fit the remaining model budget once the request itself is paid for, so the report is written from the notes instead.`
+          : 'the report turn was refused for budget with no budget figures in the refusal, so there is nothing to size a retry from; the report is written from the notes instead.');
+      }
+      if (smaller) {
+        shrunk = true;
+        transcript.write({ kind: 'report_budget_retry', attempt, max_tokens: smaller, figures: error.figures || {} });
+        log(`the report turn did not fit the remaining model budget; asking again with an output ceiling of ${smaller} tokens.`);
+        maxTokens = smaller;
+        continue;
+      }
+      // A report already written on an earlier turn is the report, with its
+      // lint, rather than a partial one rebuilt from the notes.
+      if (lastCandidate) {
+        report = lastCandidate;
+        log(`the model budget ran out on a follow-up report turn; keeping the report.json already written (${lint.length} problem(s)).`);
+        break;
+      }
+      throw error;
+    }
+    ledger.record({ model: answer.model, usage: answer.usage, ms: Date.now() - askedAt });
     transcript.write({ kind: 'report_turn', attempt, stop_reason: answer.stopReason, files: answer.toolUses.map(t => t.input && t.input.path) });
 
     const files = new Map();
@@ -737,7 +833,22 @@ async function reportPhase({
     if (normalised && normalised.changed > 0) {
       transcript.write({ kind: 'report_evidence_normalised', attempt, entries: normalised.changed });
     }
-    const candidate = normalised ? capNetworkOnlyFindings(stampReport(normalised.report, facts), { notes, screenshots }) : null;
+    const capped = normalised ? capNetworkOnlyFindings(stampReport(normalised.report, facts), { notes, screenshots }) : null;
+    const runnerDrop = capped ? dropRunnerRejectedFindings(capped, { notes, observations }) : { report: null, dropped: [] };
+    for (const entry of runnerDrop.dropped) {
+      transcript.write({ kind: 'finding_dropped', id: entry.finding.id, severity: entry.finding.severity, title: entry.finding.title, reason: entry.reason });
+    }
+    droppedByRunner = runnerDrop.dropped;
+    const contradicted = runnerDrop.report ? downgradeContradictedFindings(runnerDrop.report, play.actionLog) : { report: null, downgraded: [] };
+    const candidate = contradicted.report;
+    for (const entry of contradicted.downgraded) {
+      transcript.write({ kind: 'finding_downgraded', id: entry.id, from: entry.from, to: entry.to, control: entry.control, action: entry.action });
+    }
+    downgradedByRunner = contradicted.downgraded;
+    if (contradicted.downgraded.length > 0) {
+      log(`downgraded ${contradicted.downgraded.length} finding(s) the action log contradicts: ${contradicted.downgraded.map(entry => `${entry.id} (${entry.from} to ${entry.to}, ${entry.control} clicked at step ${entry.action})`).join(', ')}`);
+    }
+    if (candidate) lastCandidate = candidate;
     const validation = candidate ? validator.validate(candidate) : { ok: false, details: [{ instancePath: '/', message: parseError || 'report.json was not written' }] };
     const evidenceProblems = candidate ? await checkEvidence(candidate, runDirectory) : [];
     // The server lints the same four sections at `complete` and records the
@@ -798,6 +909,12 @@ async function reportPhase({
   if (!markdown) {
     markdown = renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason: 'The report step wrote report.json but no report.md.' });
   }
+  if (droppedByRunner.length > 0) {
+    markdown = rewriteMarkdown(markdown, droppedByRunner.map(entry => entry.finding), report.findings || [], RUNNER_DROPPED_SECTION);
+    log(`dropped ${droppedByRunner.length} finding(s) the runner's evidence rules refuse: ${droppedByRunner.map(entry => `${entry.finding.id} (${entry.reason})`).join(', ')}`);
+  }
+  // The uploaded pair agree: report.md says the severity report.json does.
+  if (downgradedByRunner.length > 0) markdown = rewriteDowngradedMarkdown(markdown, downgradedByRunner);
   const generatedAt = new Date().toISOString();
   const facts = reportFacts({ persona, driverName, target, play });
   markdown = stampGeneratedAt(markdown, generatedAt, facts);
@@ -808,6 +925,72 @@ async function reportPhase({
   return { report, markdown, lint };
 }
 
+/** The smallest output ceiling worth retrying a report turn with. */
+export const REPORT_MIN_TOKENS = 1536;
+
+/** Whether a 402 carried the three figures a retry is sized from. */
+export function hasBudgetFigures(error) {
+  const figures = (error && error.figures) || {};
+  return ['usd_used', 'usd_max', 'projected_usd'].every(key => Number.isFinite(Number(figures[key])) && figures[key] !== null && figures[key] !== '');
+}
+
+/** Characters per input token, for estimating the input share of a projection. */
+export const CHARS_PER_INPUT_TOKEN = 3.5;
+
+/**
+ * Output price per token over input price per token, assumed when the 402
+ * does not split its projection. Prices are server side only; this ratio is
+ * only used to divide the server's own projected figure into two parts.
+ */
+export const ASSUMED_OUTPUT_PRICE_RATIO = 5;
+
+/**
+ * The output ceiling a refused report turn could fit in, or null when even
+ * the minimum report cannot fit.
+ *
+ * The proxy's projection prices the input and the output ceiling together,
+ * and only the output ceiling can move, so the input's share is taken out of
+ * the room first: from the 402's own split (`projected: { input_cents,
+ * output_cents }`) when the server sends one, otherwise estimated from the
+ * request's size (characters / 3.5 at an input price a fifth of the output
+ * price). What is left of the room is then shared out over the output, with
+ * a tenth of headroom.
+ *
+ * @param {ProxyError} error
+ * @param {number} current the output ceiling that was refused
+ * @param {number} [bodyChars] the refused request's JSON size
+ * @returns {number|null}
+ */
+export function reportTokensThatFit(error, current, bodyChars = 0) {
+  const figures = (error && error.figures) || {};
+  const used = Number(figures.usd_used);
+  const max = Number(figures.usd_max);
+  const projected = Number(figures.projected_usd);
+  if (![used, max, projected].every(Number.isFinite) || projected <= 0 || current <= 0) return null;
+  const room = max - used;
+  if (room <= 0 || room >= projected) return null;
+  const split = figures.projected && typeof figures.projected === 'object' ? figures.projected : null;
+  let inputUsd;
+  let outputUsd;
+  if (split && Number.isFinite(Number(split.input_cents)) && Number.isFinite(Number(split.output_cents))) {
+    inputUsd = Number(split.input_cents) / 100;
+    outputUsd = Number(split.output_cents) / 100;
+  } else {
+    // The request's JSON size counts any base64 screenshot as text, which
+    // overstates the input (an image is billed far below its base64 length
+    // as text). Overstating the input understates what the output can have,
+    // so the estimate errs on the safe side: a retry it allows should fit.
+    const inputTokens = Math.max(0, Number(bodyChars) || 0) / CHARS_PER_INPUT_TOKEN;
+    const perInputToken = projected / (inputTokens + ASSUMED_OUTPUT_PRICE_RATIO * current);
+    inputUsd = perInputToken * inputTokens;
+    outputUsd = projected - inputUsd;
+  }
+  const outputRoom = room - inputUsd;
+  if (outputRoom <= 0 || outputUsd <= 0) return null;
+  const tokens = Math.floor(current * (outputRoom / outputUsd) * 0.9);
+  return tokens >= REPORT_MIN_TOKENS ? tokens : null;
+}
+
 /**
  * `callStepWithRetry`, plus the one retry that is about our own request
  * rather than the server's mood: `too_many_images` means this step allows
@@ -841,7 +1024,8 @@ async function callWithImageRetry({ imageBudget, keepObservations, messages, onI
  * product where the same few thousand tokens are re-sent tens of times. The
  * proxy allowlists `system` and passes the blocks through unchanged.
  */
-export function buildSystem({ skill, persona, brief, intentNotes, driverName, target, maxActions }) {
+export function buildSystem({ skill, persona, brief, intentNotes, driverName, target, maxActions, runId = null }) {
+  const identity = runIdentity({ persona, runId });
   return [
     { type: 'text', text: skill },
     { type: 'text', text: `<persona_file>\n${persona.body}\n</persona_file>` },
@@ -852,13 +1036,55 @@ export function buildSystem({ skill, persona, brief, intentNotes, driverName, ta
         `target: ${target}`,
         `max_actions: ${maxActions}`,
         `<expectations_brief>\n${briefForPrompt(brief)}\n</expectations_brief>`,
-        `<design_intent_notes>\n${intentNotesForPrompt(brief, intentNotes)}\n</design_intent_notes>`
+        `<design_intent_notes>\n${intentNotesForPrompt(brief, intentNotes)}\n</design_intent_notes>`,
+        ...(identity ? [`<run_identity>\n${runIdentityForPrompt(identity)}\n</run_identity>`] : [])
       ].join('\n'),
       cache_control: { type: 'ephemeral' }
     }
   ];
 }
 
+/**
+ * The run id cut to something short enough for an email or a display name:
+ * the last eight letters and digits after the `pt_run_` prefix.
+ *
+ * @param {string} runId
+ * @returns {string}
+ */
+export function shortRunId(runId) {
+  const compact = String(runId == null ? '' : runId).replace(/^pt_run_/, '').toLowerCase().replace(/[^a-z0-9]/g, '');
+  return compact.slice(-8);
+}
+
+/**
+ * A fresh identity for every run, so a sign-up form never sees an email an
+ * earlier run already registered. The Texas round lost actions to a 409 on
+ * sign-up from a reused address. `example.test` is a reserved name that never
+ * delivers mail.
+ *
+ * @param {{persona: {slug?: string, name?: string}, runId: string}} args
+ * @returns {{email: string, displayName: string, short: string}|null}
+ */
+export function runIdentity({ persona, runId }) {
+  const short = shortRunId(runId);
+  if (!short) return null;
+  const slug = String((persona && persona.slug) || 'player').toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 40) || 'player';
+  const name = String((persona && (persona.name || persona.slug)) || 'Player').replace(/\s+/g, ' ').trim().slice(0, 40) || 'Player';
+  return { email: `persona-${slug}-${short}@example.test`, displayName: `${name} ${short}`, short };
+}
+
+/**
+ * @param {{email: string, displayName: string}} identity
+ * @returns {string}
+ */
+export function runIdentityForPrompt(identity) {
+  return [
+    `email: ${identity.email}`,
+    `display_name: ${identity.displayName}`,
+    'Use this email and this display name whenever a form asks for an email, a name, a username or a coach or player name, unless the brief says otherwise. They are new for this run, so an account made with them does not exist yet. Never reuse an email from an earlier run.'
+  ].join('\n');
+}
+
 /**
  * The brief as the persona skill expects it: the `fields` object the
  * expectations-brief schema describes, with the version alongside so a report
@@ -980,6 +1206,7 @@ async function finish(outcome, ledger, runDirectory, transcript, heartbeat) {
     }
   }
   outcome.usage = ledger.toReportUsage();
+  outcome.duration = ledger.duration();
   await writeUsageFile(runDirectory, ledger);
   transcript.write({ kind: 'end', state: outcome.state, quit_reason: outcome.quitReason, actions: outcome.actionsTaken });
   await transcript.close();
@@ -995,6 +1222,7 @@ async function finish(outcome, ledger, runDirectory, transcript, heartbeat) {
 export function renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason }) {
   const good = notes.filter(note => note.kind === 'good');
   const bad = notes.filter(note => note.kind === 'bug' || note.kind === 'confusing');
+  const network = notes.filter(note => note.automatic && note.url);
   const list = entries => (entries.length > 0
     ? entries.map(note => `- **${note.kind}${note.severity ? `/${note.severity}` : ''}**: ${note.text}`).join('\n')
     : '- Nothing was logged.');
@@ -1022,6 +1250,7 @@ export function renderFallbackMarkdown({ persona, driverName, target, notes, scr
     '## Suggested next step',
     'Run this persona again once the reason above is resolved.',
     '',
+    ...(network.length > 0 ? ['## Network notes', network.map(note => `- ${note.status ? `${note.status} ` : ''}${note.url}`).join('\n'), ''] : []),
     '## Evidence',
     screenshots.length > 0 ? screenshots.map(shot => `- ${shot.relativePath}`).join('\n') : '- (none)',
     ''
@@ -1068,6 +1297,8 @@ export function notesToFindings(notes, screenshots) {
   for (const note of notes) {
     const category = NOTE_CATEGORY[note.kind];
     if (!category || !note.text) continue;
+    // A blocker or major written on a driver error is the runner's failure.
+    if (note.driver_error && ABOVE_MINOR.includes(note.severity)) continue;
     const text = String(note.text);
     const shot = nearestScreenshot(note, screenshots);
     findings.push({
@@ -1145,6 +1376,7 @@ async function writePartialMarkdown({ runDirectory, persona, driverName, target,
 export function noteLine(note, i) {
   const tags = [
     note.automatic ? 'automatic' : null,
+    note.driver_error ? 'written on a driver error' : null,
     Number.isInteger(note.observation) && note.observation > 0 ? `seen at observation ${note.observation}` : null,
     note.screenshot ? `screenshot ${note.screenshot}` : null
   ].filter(Boolean);
@@ -1182,7 +1414,7 @@ export const MAX_OBSERVATIONS_IN_REPORT = 200;
 function observationLines(observations) {
   if (!observations || observations.length === 0) return '(none)';
   const shown = observations.slice(-MAX_OBSERVATIONS_IN_REPORT);
-  const lines = shown.map(entry => `observation ${entry.index}: step ${entry.step}${entry.url ? ` ${entry.url}` : ''}${entry.title ? ` "${entry.title}"` : ''}`);
+  const lines = shown.map(entry => `observation ${entry.index}: step ${entry.step}${entry.url ? ` ${entry.url}` : ''}${entry.title ? ` "${entry.title}"` : ''}${entry.driver_error ? ` [driver error: ${entry.driver_error}]` : ''}`);
   if (shown.length < observations.length) lines.unshift(`(the first ${observations.length - shown.length} are left out)`);
   return lines.join('\n');
 }
@@ -1215,6 +1447,332 @@ export function observationTitle(observation) {
  * @param {{notes: Array<Object>, screenshots: Array<Object>}} context
  * @returns {Object} the report, with capped findings rewritten
  */
+/** How report.md lists what the runner dropped. */
+export const RUNNER_DROPPED_SECTION = Object.freeze({
+  heading: '## Dropped by the runner (driver errors or waiting notes only)',
+  explanation: 'The runner removes a blocker or major whose only cited observations are driver errors (the runner could not carry out an action; the game did nothing wrong), and any finding cited only by waiting notes. They are not sent to the server. They are kept here so you can judge them yourself.',
+  emptyNote: '- Nothing here that the runner kept. See the dropped findings below.'
+});
+
+/**
+ * The findings the runner refuses before anything is uploaded, and the
+ * report without them.
+ *
+ * - A blocker or major whose transcript evidence (observations and notes it
+ *   cites) all points at driver errors. Screenshots do not rescue it: the
+ *   Texas run filed "the season loop is fully blocked" from three strict mode
+ *   failures, with a screenshot of the screen working.
+ * - Any finding whose transcript evidence is only waiting notes.
+ *
+ * A finding with no transcript evidence is left to the other checks.
+ *
+ * @param {Object} report
+ * @param {{notes?: Array, observations?: Array}} context
+ * @returns {{report: Object, dropped: Array<{finding: Object, reason: string}>}}
+ */
+export function dropRunnerRejectedFindings(report, { notes = [], observations = [] } = {}) {
+  if (!report || !Array.isArray(report.findings)) return { report, dropped: [] };
+  const byIndex = new Map(observations.map(entry => [entry.index, entry]));
+  const verdict = finding => {
+    const cited = [];
+    for (const item of Array.isArray(finding.evidence) ? finding.evidence : []) {
+      if (!item || item.type !== 'transcript_step') continue;
+      const ref = String(item.ref || '');
+      const observation = ref.match(/^\s*observation\s*#?\s*(\d+)\s*$/i);
+      const note = ref.match(/^\s*note\s*#?\s*(\d+)\s*$/i);
+      if (observation) {
+        const entry = byIndex.get(Number(observation[1]));
+        cited.push(entry && entry.driver_error ? 'driver' : 'other');
+      } else if (note) {
+        const entry = notes[Number(note[1]) - 1];
+        cited.push(entry && entry.kind === 'waiting' ? 'waiting' : entry && entry.driver_error ? 'driver' : 'other');
+      } else {
+        cited.push('other');
+      }
+    }
+    if (cited.length === 0) return null;
+    if (ABOVE_MINOR.includes(finding.severity) && cited.every(kind => kind === 'driver')) return 'driver errors only';
+    if (cited.every(kind => kind === 'waiting')) return 'waiting notes only';
+    if (ABOVE_MINOR.includes(finding.severity) && cited.every(kind => kind === 'driver' || kind === 'waiting')) return 'driver errors and waiting notes only';
+    return null;
+  };
+  const dropped = [];
+  const findings = [];
+  for (const finding of report.findings) {
+    const reason = finding ? verdict(finding) : null;
+    if (reason) dropped.push({ finding, reason });
+    else findings.push(finding);
+  }
+  return { report: dropped.length > 0 ? { ...report, findings } : report, dropped };
+}
+
+/** The most actions the report prompt lists; the newest are kept. */
+export const MAX_ACTIONS_IN_REPORT = 60;
+
+/** The longest typed text an action line repeats. */
+const MAX_TYPED_CHARS = 40;
+
+/**
+ * The names of the refs a compacted snapshot shows: `- button "Start" [e12]`
+ * gives e12 -> button "Start".
+ *
+ * @param {string} text
+ * @returns {Map<string, {role: string, name: string}>}
+ */
+export function refNamesFrom(text) {
+  const names = new Map();
+  for (const line of String(text == null ? '' : text).split('\n')) {
+    const match = line.match(/^\s*- ([a-z]+) "([^"]*)"(?: \[[^\]]*\])*? \[(e\d+)\]/);
+    if (match && match[2]) names.set(match[3], { role: match[1], name: match[2] });
+  }
+  return names;
+}
+
+/**
+ * Every action that went through, from the transcript's observation and
+ * action lines, with the name of what it targeted when the screen before it
+ * showed one.
+ *
+ * @param {Array<Object>} records transcript lines of kind observation and action, in order
+ * @returns {Array<{action: number, kind: string, target: string, name: string|null, ref: string|null, changed: boolean}>}
+ */
+export function actionLogFrom(records) {
+  const log = [];
+  let names = new Map();
+  // Observations are numbered as the report prompt numbers them: one per
+  // observation line, in order.
+  let observation = 0;
+  let previousText = '';
+  let pending = null;
+  for (const record of Array.isArray(records) ? records : []) {
+    if (!record) continue;
+    if (record.kind === 'observation') {
+      observation += 1;
+      const text = String(record.text == null ? '' : record.text);
+      if (pending) {
+        pending.observationAfter = observation;
+        pending.introducedError = introducesError(previousText, text);
+        pending = null;
+      }
+      previousText = text;
+      const next = refNamesFrom(text);
+      // An unchanged marker or a canvas shows no refs; the last names stand.
+      if (next.size > 0) names = next;
+      continue;
+    }
+    if (record.kind !== 'action') continue;
+    pending = null;
+    if (!record.ok || !ACTION_TOOLS.includes(record.tool)) continue;
+    const input = record.input || {};
+    const ref = typeof input.ref === 'string' && input.ref.trim() ? input.ref.trim() : null;
+    const known = ref ? names.get(ref) : null;
+    let name = known ? known.name : null;
+    let target;
+    if (record.tool === 'key') target = `key ${String(input.key || '')}`;
+    else if (record.tool === 'navigate') target = String(input.url || '');
+    else if (record.tool === 'swipe') target = `from (${input.x},${input.y}) to (${input.toX},${input.toY})`;
+    else {
+      const named = typeof input.target === 'string' && input.target.trim() ? input.target.trim() : null;
+      if (!name && named) {
+        const roleAndName = named.match(/^[a-z]+\s+"([^"]*)"$/);
+        name = roleAndName ? roleAndName[1] : named;
+      }
+      const where = known ? `${known.role} "${known.name}" [${ref}]` : ref ? `[${ref}]` : named || (Number.isFinite(Number(input.x)) && Number.isFinite(Number(input.y)) ? `(${input.x},${input.y})` : 'the focused element');
+      if (record.tool === 'type') {
+        const text = String(input.text == null ? '' : input.text);
+        target = `${JSON.stringify(text.length > MAX_TYPED_CHARS ? `${text.slice(0, MAX_TYPED_CHARS - 3)}...` : text)} into ${where}${input.replace ? ' (replacing)' : ''}${input.submit ? ' then Enter' : ''}`;
+      } else {
+        target = where;
+      }
+    }
+    const entry = { action: Number(record.actions) || log.length + 1, kind: record.tool, target, name, ref, changed: Boolean(record.changed), observationBefore: observation, observationAfter: null, introducedError: false };
+    log.push(entry);
+    pending = entry;
+  }
+  return log;
+}
+
+/** A line that reads as an error: an alert or a dialog, or error wording. */
+const ERROR_LINE = /^\s*- (?:alert|alertdialog|dialog)\b|\b(?:error|errors|failed|invalid)\b/i;
+
+/**
+ * Whether the screen after an action shows an alert, a dialog or error
+ * wording that the screen before it did not: a change, but not the one the
+ * persona wanted.
+ *
+ * @param {string} before
+ * @param {string} after
+ * @returns {boolean}
+ */
+export function introducesError(before, after) {
+  const seen = new Set(String(before || '').split('\n').map(line => line.trim()));
+  return String(after || '').split('\n').some(line => !seen.has(line.trim()) && ERROR_LINE.test(line));
+}
+
+/**
+ * The action log as the report prompt's numbered list, bounded to the newest
+ * {@link MAX_ACTIONS_IN_REPORT}.
+ *
+ * @param {Array<Object>} log
+ * @returns {string}
+ */
+export function actionLogLines(log) {
+  if (!Array.isArray(log) || log.length === 0) return '(none)';
+  const shown = log.slice(-MAX_ACTIONS_IN_REPORT);
+  const lines = shown.map(entry => `${entry.action}. ${entry.kind} ${entry.target}: ${entry.changed ? 'screen changed' : 'no change'}`);
+  if (shown.length < log.length) lines.unshift(`(the first ${log.length - shown.length} are left out)`);
+  return lines.join('\n');
+}
+
+/**
+ * The failed requests the runner recorded, one line each, for the report
+ * prompt's network notes.
+ *
+ * @param {Array<Object>} notes
+ * @returns {string}
+ */
+export function networkNoteLines(notes) {
+  const lines = [];
+  (notes || []).forEach((note, index) => {
+    if (!note || !note.automatic || !note.url) return;
+    lines.push(`- note ${index + 1}: ${note.status ? `${note.status} ` : ''}${note.url}`);
+  });
+  return lines.length > 0 ? lines.join('\n') : '(none)';
+}
+
+/**
+ * The "nothing happens" family: wording that says a control was used and
+ * produced nothing. Deliberately narrow; "cannot" or "fails" can be true of a
+ * control the persona clicked.
+ */
+const NOTHING_HAPPENED = /\b(?:did nothing|does nothing|never kicked off|never started|no response|nothing happened)\b/i;
+
+/** One step down the severity scale. Cosmetic and praise are never touched. */
+const DOWNGRADE = Object.freeze({ blocker: 'major', major: 'minor', minor: 'cosmetic' });
+
+/**
+ * Whether a text names a control as a whole word, ignoring case: "Play"
+ * matches "press Play" and never "Players".
+ *
+ * @param {string} text
+ * @param {string} name
+ * @returns {boolean}
+ */
+export function namesControl(text, name) {
+  const value = String(name || '').trim();
+  if (value.length < 3) return false;
+  const escaped = value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  return new RegExp(`(?<![\\p{L}\\p{N}])${escaped}(?![\\p{L}\\p{N}])`, 'iu').test(String(text || ''));
+}
+
+/**
+ * Downgrade, never drop, a finding the action log contradicts. All of these
+ * have to hold, or the severity is left alone:
+ *
+ * - its title or description is in the "nothing happens" family ("did
+ *   nothing", "does nothing", "never kicked off", "never started", "no
+ *   response", "nothing happened") and names a control, whole word, of at
+ *   least three characters;
+ * - the most recent click on that control changed the screen, and the screen
+ *   after it gained no alert, dialog or error wording;
+ * - that click happened at or after the latest observation the finding cites
+ *   (a finding citing no observation is left alone).
+ *
+ * The Texas round's report said the season was never kicked off, with the
+ * click on KICK OFF THE SEASON and a changed screen in its own transcript.
+ *
+ * @param {Object} report
+ * @param {Array<Object>} log from {@link actionLogFrom}
+ * @returns {{report: Object, downgraded: Array<{id: string, from: string, to: string, control: string, action: number, note: string}>}}
+ */
+export function downgradeContradictedFindings(report, log) {
+  if (!report || !Array.isArray(report.findings) || !Array.isArray(log) || log.length === 0) return { report, downgraded: [] };
+  const clicks = log.filter(entry => entry.kind === 'click' && entry.name && entry.name.trim().length >= 3);
+  if (clicks.length === 0) return { report, downgraded: [] };
+  const downgraded = [];
+  const findings = report.findings.map(finding => {
+    if (!finding || !DOWNGRADE[finding.severity]) return finding;
+    const text = `${finding.title || ''}\n${finding.description || ''}`;
+    if (!NOTHING_HAPPENED.test(text)) return finding;
+    const cited = (Array.isArray(finding.evidence) ? finding.evidence : [])
+      .map(item => (item && item.type === 'transcript_step' ? String(item.ref || '').match(/^\s*observation\s*#?\s*(\d+)\s*$/i) : null))
+      .filter(Boolean)
+      .map(match => Number(match[1]));
+    if (cited.length === 0) return finding;
+    const citedLatest = Math.max(...cited);
+    // The controls the finding names, each judged by its most recent click.
+    const named = [...new Set(clicks.map(entry => entry.name.trim().toLowerCase()))]
+      .filter(name => namesControl(text, name));
+    let hit = null;
+    for (const name of named) {
+      const latest = clicks.filter(entry => entry.name.trim().toLowerCase() === name).at(-1);
+      if (!latest.changed || latest.introducedError) continue;
+      if (!Number.isInteger(latest.observationBefore) || latest.observationBefore < citedLatest) continue;
+      hit = latest;
+      break;
+    }
+    if (!hit) return finding;
+    const to = DOWNGRADE[finding.severity];
+    const note = `the action log shows ${hit.name.trim()} clicked at step ${hit.action} with the screen changing`;
+    const description = String(finding.description || '');
+    downgraded.push({ id: finding.id, from: finding.severity, to, control: hit.name.trim(), action: hit.action, note });
+    return { ...finding, severity: to, description: description ? `${description}\n\nDowngraded by the runner: ${note}.` : `Downgraded by the runner: ${note}.` };
+  });
+  return { report: downgraded.length > 0 ? { ...report, findings } : report, downgraded };
+}
+
+/** How report.md lists the findings the runner downgraded. */
+export const DOWNGRADED_HEADING = '## Downgraded by the runner (the action log contradicts the claim)';
+
+/**
+ * report.md agreeing with report.json about downgraded findings: on every
+ * line that names a downgraded finding's id, its old severity word becomes
+ * the new one, and a section lists each downgrade with the runner's note,
+ * replacing one written before.
+ *
+ * @param {string} markdown
+ * @param {Array<{id: string, from: string, to: string, note: string}>} downgraded
+ * @returns {string}
+ */
+export function rewriteDowngradedMarkdown(markdown, downgraded) {
+  if (!Array.isArray(downgraded) || downgraded.length === 0) return markdown;
+  const text = String(markdown || '');
+  const at = text.indexOf(`\n${DOWNGRADED_HEADING}`);
+  let body = (at === -1 ? (text.startsWith(DOWNGRADED_HEADING) ? '' : text) : text.slice(0, at)).replace(/\s+$/, '');
+  const recase = (word, to) => (word === word.toUpperCase() ? to.toUpperCase() : word[0] === word[0].toUpperCase() ? to[0].toUpperCase() + to.slice(1) : to);
+  const escape = value => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  const namesId = (line, id) => new RegExp(`(^|[^A-Za-z0-9])${escape(id)}(?![0-9])`).test(line);
+  // The finding a "Severity:" line belongs to: the one named by the nearest
+  // heading or list item above it. A heading naming none ends it.
+  let current = null;
+  body = body.split('\n').map(line => {
+    let out = line;
+    const named = downgraded.find(entry => entry.id && namesId(line, String(entry.id)));
+    if (named) current = named;
+    else if (/^#{1,6} /.test(line) || /^\s*(?:[-*+]|\d+[.)])\s+\S*\b[A-Z]\d{2,}\b/.test(line)) current = null;
+    for (const entry of downgraded) {
+      const id = String(entry.id || '');
+      if (!id) continue;
+      // Only the severity slot the writer labels: right after the id in
+      // parentheses or brackets ("F001 (major)", "**F001 [Major, bug]**"), or
+      // after "Severity:" within that finding. A title that happens to
+      // contain the word stays as written.
+      if (namesId(out, id)) {
+        out = out.replace(new RegExp(`(${escape(id)}\\**\\s*[([]\\s*)(${entry.from})\\b`, 'i'), (match, lead, word) => `${lead}${recase(word, entry.to)}`);
+      }
+      if (current === entry) {
+        out = out.replace(new RegExp(`(severity\\**\\s*:\\s*\\**\\s*)(${entry.from})\\b`, 'gi'), (match, lead, word) => `${lead}${recase(word, entry.to)}`);
+      }
+    }
+    return out;
+  }).join('\n');
+  const section = [
+    DOWNGRADED_HEADING,
+    ...downgraded.map(entry => `- ${entry.id}: ${entry.from} to ${entry.to}, because ${entry.note}.`)
+  ].join('\n');
+  return `${body}\n\n${section}\n`;
+}
+
 export function capNetworkOnlyFindings(report, { notes = [], screenshots = [] } = {}) {
   if (!report || !Array.isArray(report.findings)) return report;
   const byShot = new Map();