ravensight-playtest 0.3.1 → 0.3.2

package/src/run/noise.js

@@ -0,0 +1,102 @@
+/**
+ * Console noise a developer expects, which is never a finding.
+ *
+ * Two sources: the built in rule (a 401 on a who am I check is what a
+ * signed out page does when it asks "who am I?"; the Snowball round filed it
+ * as a bug), and the brief's own `expected_noise` patterns, which live in the
+ * local brief (see ../briefLocal.js).
+ *
+ * A pattern is matched against an automatic console note:
+ *
+ * - `/.../` is a case insensitive regular expression over the note's text and URL;
+ * - otherwise any status codes (`401`) and paths (`/auth/me`) in it must all
+ *   appear in the note (its status, URL or text), and the other words are a
+ *   description for the reader, such as "before login";
+ * - a pattern with neither is a case insensitive substring of the note's text.
+ */
+
+/** The built in rule, as the prompt and the README state it. */
+export const BUILT_IN_NOISE = 'a 401 on a who am I check, a request whose last path segment is /me, /whoami, /session or /current_user such as /auth/me, while signed out is not a finding';
+
+/**
+ * A request whose final path segment asks who the visitor is: `/auth/me`,
+ * `/me`, `/whoami`, `/session`, `/current_user`. Only the last segment counts,
+ * so `/me/inventory`, `/user/42/save`, `/account/purchase`, `/token` and
+ * `/login` are never this rule's: a 401 there can be a real failure.
+ */
+const AUTH_CHECK = /\/(?:me|whoami|session|current[_-]?user)\/?(?=$|[?#\s)'",])/i;
+
+/**
+ * @param {Object} note
+ * @returns {string}
+ */
+function haystack(note) {
+  return `${note && note.text ? note.text : ''} ${note && note.url ? note.url : ''}`;
+}
+
+/**
+ * Whether a note is the built in case: a 401 on a who am I check.
+ *
+ * @param {Object} note
+ * @returns {boolean}
+ */
+export function isAuthCheck401(note) {
+  if (!note) return false;
+  const is401 = note.status === 401 || /\b401\b/.test(String(note.text || ''));
+  return is401 && AUTH_CHECK.test(haystack(note));
+}
+
+/**
+ * Whether one pattern matches a note.
+ *
+ * @param {Object} note
+ * @param {string} pattern
+ * @returns {boolean}
+ */
+export function matchesPattern(note, pattern) {
+  const value = String(pattern == null ? '' : pattern).trim();
+  if (!value || !note) return false;
+  const text = haystack(note);
+  const regex = value.match(/^\/(.+)\/([a-z]*)$/);
+  if (regex) {
+    try {
+      return new RegExp(regex[1], regex[2].includes('i') ? regex[2] : `${regex[2]}i`).test(text);
+    } catch {
+      return false;
+    }
+  }
+  const statuses = value.match(/\b[1-5]\d\d\b/g) || [];
+  const paths = (value.match(/(?:^|\s)(\/[^\s]*)/g) || []).map(entry => entry.trim().toLowerCase());
+  if (statuses.length === 0 && paths.length === 0) return text.toLowerCase().includes(value.toLowerCase());
+  const lower = text.toLowerCase();
+  return statuses.every(code => note.status === Number(code) || new RegExp(`\\b${code}\\b`).test(text))
+    && paths.every(entry => lower.includes(entry));
+}
+
+/**
+ * Whether an automatic note is expected noise. A persona's own note never is:
+ * if the persona saw the game break, that is a finding whatever the console said.
+ *
+ * @param {Object} note
+ * @param {Array<string>} [patterns] the brief's expected_noise
+ * @returns {boolean}
+ */
+export function isExpectedNoise(note, patterns = []) {
+  if (!note || !note.automatic) return false;
+  if (isAuthCheck401(note)) return true;
+  return (Array.isArray(patterns) ? patterns : []).some(pattern => matchesPattern(note, pattern));
+}
+
+/**
+ * The expected noise block of the persona prompt.
+ *
+ * @param {Array<string>} [patterns]
+ * @returns {string}
+ */
+export function expectedNoiseForPrompt(patterns = []) {
+  const lines = [`- built in: ${BUILT_IN_NOISE}`];
+  for (const pattern of Array.isArray(patterns) ? patterns : []) if (typeof pattern === 'string' && pattern.trim()) lines.push(`- ${pattern.trim()}`);
+  return lines.join('\n');
+}
+
+export default { isExpectedNoise, isAuthCheck401, matchesPattern, expectedNoiseForPrompt, BUILT_IN_NOISE };

package/src/run/personaLoop.js

@@ -1,17 +1,19 @@
 import { rewriteMarkdown } from './dropped.js';
 import { selectAction, JEV_CONTROLLER_PROMPT } from './actionSelector.js';
-import { mkdir, writeFile } from 'node:fs/promises';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import { prepareMessages, MAX_IMAGES_IN_CONTEXT, KEEP_OBSERVATIONS, shrinkImages } from './context.js';
 import { withRollingCacheBreakpoint } from './fixLoop.js';
 import { callStepWithRetry, ProxyError, STEPS } from './model.js';
 import { RUN_FILES } from './paths.js';
-import { checkEvidence, createReportValidator, lintMarkdownSections, normaliseEvidence, repairInstruction, REQUIRED_MARKDOWN_SECTIONS, stampReport, writeReportFiles } from './report.js';
+import { appendNormalisedMarkdown, checkEvidence, createReportValidator, lintMarkdownSections, normaliseEnums, normaliseEvidence, repairInstruction, REQUIRED_MARKDOWN_SECTIONS, stampReport, writeReportFiles } from './report.js';
 import { assertNoSecrets } from './secretScan.js';
 import { ACTION_TOOLS, PLAY_TOOLS, SWIPE_TOOL, REPORT_TOOLS, executeTool } from './tools.js';
 import { createTranscript } from './transcript.js';
 import { createUsageLedger, writeUsageFile } from './usage.js';
 import { fillPlaceholders, fillReportPlaceholders } from './placeholders.js';
+import { BUILT_IN_NOISE, expectedNoiseForPrompt, isExpectedNoise } from './noise.js';
+import { MIN_PASSWORD_CHARS } from '../briefLocal.js';
 
 /**
  * One persona's playthrough: PLAYING then REPORTING (spec 03).
@@ -130,6 +132,9 @@ export async function runPersona(args) {
     runDirectory,
     skill,
     brief = null,
+    // The brief fields that stay on this machine (../briefLocal.js):
+    // expected_noise and test_account.
+    localBrief = {},
     intentNotes = [],
     schemas,
     maxActions,
@@ -267,6 +272,7 @@ export async function runPersona(args) {
       validator,
       skill: args.reportSkill || skill,
       brief,
+      localBrief,
       intentNotes,
       transcript,
       ledger,
@@ -330,6 +336,7 @@ async function playLoop({
   target,
   skill,
   brief,
+  localBrief = {},
   intentNotes,
   maxActions,
   transcript,
@@ -344,8 +351,10 @@ async function playLoop({
   onBudgetWarning = null,
   log = () => {}
 }) {
-  const system = buildSystem({ skill, persona, brief, intentNotes, driverName, target, maxActions, runId });
+  const system = buildSystem({ skill, persona, brief, localBrief, intentNotes, driverName, target, maxActions, runId });
   const identity = runIdentity({ persona, runId });
+  const secrets = secretsOf(localBrief);
+  const noisePatterns = (localBrief && Array.isArray(localBrief.expected_noise)) ? localBrief.expected_noise : [];
   if (controller === 'jev') system.push({ type: 'text', text: JEV_CONTROLLER_PROMPT });
   const availableTools = driver.supportedActions
     ? [...PLAY_TOOLS, ...(driver.supportedActions.includes('swipe') ? [SWIPE_TOOL] : [])].filter(tool => !ACTION_TOOLS.includes(tool.name) || driver.supportedActions.includes(tool.name))
@@ -391,7 +400,9 @@ async function playLoop({
     return entry;
   };
   const noted = note => {
-    const entry = { ...note, at_action: actions, observation: observations.length, order: nextOrder() };
+    // A note never carries the test account password, in the transcript, a
+    // partial report or the report prompt.
+    const entry = { ...scrubSecrets(note, secrets), at_action: actions, observation: observations.length, order: nextOrder() };
     notes.push(entry);
     awaitingAfter.push(entry);
     return entry;
@@ -520,7 +531,7 @@ async function playLoop({
           // overwrite the transcript line's `kind`, and every note would
           // read as a different sort of line.
           noted(note);
-          transcript.write({ kind: 'note', tool: toolUse.name, note });
+          transcript.write({ kind: 'note', tool: toolUse.name, note: scrubSecrets(note, secrets) });
         },
         onScreenshot: shot => {
           screenshots.push({ ...shot, observation: observations.length, order: nextOrder() });
@@ -533,8 +544,8 @@ async function playLoop({
       const actionLine = {
         kind: 'action',
         tool: toolUse.name,
-        input: redactToolInput(toolUse),
-        thought: (toolUse.input && toolUse.input.thought_in_character) || null,
+        input: redactToolInput(toolUse, secrets),
+        thought: scrubSecrets((toolUse.input && toolUse.input.thought_in_character) || null, secrets),
         ok: !executed.error,
         error: executed.error,
         changed: executed.changed,
@@ -597,8 +608,12 @@ async function playLoop({
           ...(entry.url && Number.isInteger(entry.status) ? { status: entry.status } : {}),
           ...(shot ? { screenshot: shot.relativePath } : {})
         };
+        // Noise the developer said to expect (or the built in rule: a 401 on
+        // an auth check while signed out) is marked, so the report step is
+        // told and the runner can drop a finding that rests only on it.
+        if (isExpectedNoise({ ...note, status: Number.isInteger(entry.status) ? entry.status : note.status }, noisePatterns)) note.expected_noise = true;
         noted(note);
-        transcript.write({ kind: 'note', note });
+        transcript.write({ kind: 'note', note: scrubSecrets(note, secrets) });
       }
       asides.push(`The game logged ${consoleErrors.length} error(s): ${consoleErrors.map(entry => entry.text).join(' | ').slice(0, 800)}${shot ? ` A screenshot is saved at ${shot.relativePath}.` : ''}`);
     }
@@ -664,9 +679,11 @@ async function reportPhase({
   driverName,
   target,
   runDirectory,
+  schemas = [],
   validator,
   skill,
   brief,
+  localBrief = {},
   intentNotes,
   transcript,
   ledger,
@@ -678,7 +695,11 @@ async function reportPhase({
   endedAt,
   log
 }) {
-  const system = buildSystem({ skill, persona, brief, intentNotes, driverName, target, maxActions: play.actionsTaken, runId });
+  // The report step never sees the test account: it has nothing to sign in
+  // to, and the password crosses the model proxy during play only.
+  const secrets = secretsOf(localBrief);
+  const reportLocal = localBrief && Array.isArray(localBrief.expected_noise) ? { expected_noise: localBrief.expected_noise } : {};
+  const system = buildSystem({ skill, persona, brief, localBrief: reportLocal, intentNotes, driverName, target, maxActions: play.actionsTaken, runId });
   const messages = [{
     role: 'user',
     content: [{
@@ -693,6 +714,7 @@ async function reportPhase({
         '',
         'Notes marked automatic were recorded by the runner from the console, with no severity: decide one from what the persona saw. Network notes are context, not bugs: a failed request is a finding only when the persona saw a broken screen because of it. A finding whose only evidence is automatic network notes is capped at minor by the runner.',
         'Driver errors are never evidence of a game bug. An observation marked [driver error: ...] or a note "written on a driver error" means the runner could not carry out the action (a target that matched two elements, locator code, a ref not on screen, a timeout on a page that did not change); the game did nothing wrong there. Never report a blocked loop, a frozen screen or a broken button from them. The runner drops any blocker or major whose only cited observations are driver errors.',
+        `Notes marked expected noise match the brief's expected_noise or the built in rule (${BUILT_IN_NOISE}); they are never a finding of any severity, and the runner drops a finding whose transcript evidence is only expected noise.`,
         'Notes of kind waiting ("waiting for X to finish") are context, never a finding of any severity, and the runner drops a finding cited only by them. Before reporting a loading problem, check the later observations and screenshots: if they show the content loaded, it is not a finding.',
         '',
         'Observations the persona was shown, by index:',
@@ -707,7 +729,7 @@ async function reportPhase({
         networkNoteLines(notes),
         '',
         'Each note carries "after": the URL and title of the observation that followed it. A blocker or major finding must cite the observation index where the failure is visible, as evidence {"type": "transcript_step", "ref": "observation N"}, and must not be contradicted by its note\'s "after" state or by a later persona note: if the next screen or a later note shows the game working, it is not a blocker or a major. Cite a note as {"type": "transcript_step", "ref": "note N"}.',
-        'A blocker or major finding must also cite the screenshot taken at or after that observation (the list below says which observation each one followed), and that screenshot must not contradict it: if it shows the thing the finding says is missing or broken, such as the field or the button, it is not a blocker or a major. A text observation can be cut short on a long page; the screenshot is what the player saw.',
+        'A blocker or major finding must also cite the screenshot taken at or after that observation (the list below says which observation each one followed), and that screenshot must not contradict it: if it shows the thing the finding says is missing or broken, such as the field or the button, it is not a blocker or a major. A text observation can be cut short on a long page; the screenshot is what the player saw. After you write the report, the runner shows each blocker or major (at most three) its cited screenshot and downgrades one step any finding the image contradicts.',
         '',
         'Every evidence entry is an object with exactly two keys, "type" and "ref", and nothing else. A screenshot is {"type": "screenshot", "ref": "screenshots/04-example.png"}, never "path" or "file".',
         '',
@@ -722,6 +744,9 @@ async function reportPhase({
       ].join('\n')
     }]
   }];
+  // The prompt is built from notes, observations and the action log; none
+  // of them may carry the test account password into the report step.
+  messages[0].content[0].text = scrubSecrets(messages[0].content[0].text, secrets);
 
   let report = null;
   let markdown = null;
@@ -733,6 +758,8 @@ async function reportPhase({
   let continued = false;
   let droppedByRunner = [];
   let downgradedByRunner = [];
+  let normalisedByRunner = [];
+  const reportSchema = ((schemas || []).find(schema => schema && schema.name === 'playtest-report') || {}).content || null;
 
   // One repair at most when the schema or the evidence refs refuse what was
   // written: each turn is a metered question, and the dogfood gate asks for
@@ -833,7 +860,16 @@ async function reportPhase({
     if (normalised && normalised.changed > 0) {
       transcript.write({ kind: 'report_evidence_normalised', attempt, entries: normalised.changed });
     }
-    const capped = normalised ? capNetworkOnlyFindings(stampReport(normalised.report, facts), { notes, screenshots }) : null;
+    // Known out of range values (a category written as the severity, an
+    // unknown category) are brought back into the schema before anything
+    // judges the report, so a run is never lost to them for want of a repair
+    // turn. The server's schema stays strict.
+    const enumFixed = normalised ? normaliseEnums(normalised.report, reportSchema) : { report: null, changes: [] };
+    normalisedByRunner = enumFixed.changes;
+    if (enumFixed.changes.length > 0) {
+      transcript.write({ kind: 'report_values_normalised', attempt, changes: enumFixed.changes.map(change => ({ id: change.id, field: change.field, from: change.from, to: change.to })) });
+    }
+    const capped = normalised ? capNetworkOnlyFindings(stampReport(enumFixed.report, facts), { notes, screenshots }) : null;
     const runnerDrop = capped ? dropRunnerRejectedFindings(capped, { notes, observations }) : { report: null, dropped: [] };
     for (const entry of runnerDrop.dropped) {
       transcript.write({ kind: 'finding_dropped', id: entry.finding.id, severity: entry.finding.severity, title: entry.finding.title, reason: entry.reason });
@@ -906,6 +942,22 @@ async function reportPhase({
     });
   }
 
+  // One bounded look at the evidence: each blocker or major (at most
+  // MAX_CLAIM_CHECKS) is shown the screenshot it cites with its claim. It runs
+  // after the report is settled, so a refusal can cost the check and never
+  // the report.
+  report = scrubSecrets(report, secrets);
+  if (!shrunk) {
+    const checked = await checkClaimsAgainstScreenshots({ client, jobId, runId, report, screenshots, runDirectory, ledger, transcript, log });
+    if (checked.downgraded.length > 0) {
+      report = checked.report;
+      downgradedByRunner = [...downgradedByRunner, ...checked.downgraded];
+      log(`downgraded ${checked.downgraded.length} finding(s) their own screenshot contradicts: ${checked.downgraded.map(entry => `${entry.id} (${entry.from} to ${entry.to}, ${entry.screenshot})`).join(', ')}`);
+    }
+  } else {
+    transcript.write({ kind: 'claim_check_skipped', reason: 'the report turn needed a smaller output ceiling to fit the budget' });
+  }
+
   if (!markdown) {
     markdown = renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason: 'The report step wrote report.json but no report.md.' });
   }
@@ -915,16 +967,152 @@ async function reportPhase({
   }
   // The uploaded pair agree: report.md says the severity report.json does.
   if (downgradedByRunner.length > 0) markdown = rewriteDowngradedMarkdown(markdown, downgradedByRunner);
+  if (normalisedByRunner.length > 0) {
+    markdown = appendNormalisedMarkdown(markdown, normalisedByRunner);
+    log(`normalised ${normalisedByRunner.length} value(s) the report schema does not allow: ${normalisedByRunner.map(change => `${change.id ? `${change.id} ` : ''}${change.field} ${JSON.stringify(change.from)} to ${JSON.stringify(change.to)}`).join(', ')}`);
+  }
   const generatedAt = new Date().toISOString();
   const facts = reportFacts({ persona, driverName, target, play });
   markdown = stampGeneratedAt(markdown, generatedAt, facts);
   // The same slots in the prose the dashboard shows from report.json.
   report = fillReportPlaceholders(report, { generatedAt, ...facts });
+  // Whatever the model wrote, the password does not leave in a report.
+  markdown = scrubSecrets(markdown, secrets);
+  report = scrubSecrets(report, secrets);
 
   await writeReportFiles({ runDirectory, markdown, report });
   return { report, markdown, lint };
 }
 
+/** The most blocker or major findings whose screenshot is checked against the claim. */
+export const MAX_CLAIM_CHECKS = 3;
+
+/** Output ceiling for one claim check: the answer is one word. */
+export const CLAIM_CHECK_MAX_TOKENS = 32;
+
+/** The instruction for a claim check. */
+export const CLAIM_CHECK_SYSTEM = 'You check playtest findings against their evidence. You are shown one screenshot and one claim about it. Answer with exactly one word: yes if the image contradicts the claim, no if it does not, unclear if the image cannot settle it.';
+
+/**
+ * The screenshot a finding's claim is checked against: of those it cites, the
+ * one taken nearest after the latest observation it cites (the later of two on
+ * the same observation), else the last screenshot it cites. Null when it cites none that exists in the list.
+ *
+ * @param {Object} finding
+ * @param {Array<Object>} screenshots
+ * @returns {Object|null}
+ */
+export function citedAfterScreenshot(finding, screenshots) {
+  const evidence = Array.isArray(finding && finding.evidence) ? finding.evidence : [];
+  const byPath = new Map((screenshots || []).filter(shot => shot && shot.relativePath).map(shot => [shot.relativePath, shot]));
+  const cited = evidence.filter(item => item && item.type === 'screenshot' && byPath.has(String(item.ref))).map(item => byPath.get(String(item.ref)));
+  if (cited.length === 0) return null;
+  const observations = evidence
+    .map(item => (item && item.type === 'transcript_step' ? String(item.ref || '').match(/^\s*observation\s*#?\s*(\d+)\s*$/i) : null))
+    .filter(Boolean)
+    .map(match => Number(match[1]));
+  if (observations.length > 0) {
+    const latest = Math.max(...observations);
+    // The nearest screen after the claim: the lowest observation at or after
+    // it, and of two taken on the same observation the later one.
+    const order = new Map((screenshots || []).map((shot, index) => [shot, index]));
+    const after = cited
+      .filter(shot => Number.isInteger(shot.observation) && shot.observation >= latest)
+      .sort((a, b) => a.observation - b.observation || order.get(b) - order.get(a))[0];
+    if (after) return after;
+  }
+  return cited[cited.length - 1];
+}
+
+/**
+ * Read a claim check answer: yes, no or unclear, from its first word.
+ *
+ * @param {string} text
+ * @returns {'yes'|'no'|'unclear'}
+ */
+export function claimCheckAnswer(text) {
+  const match = String(text || '').match(/^[^A-Za-z]*(yes|no|unclear)\b/i);
+  return match ? match[1].toLowerCase() : 'unclear';
+}
+
+/**
+ * The bounded verification turn of the report step. For each blocker or
+ * major, at most {@link MAX_CLAIM_CHECKS}, the cited `after` screenshot is
+ * sent with the finding's claim and the question "does this image contradict
+ * the claim: yes, no, unclear". A yes downgrades the finding one step, with a
+ * note naming the screenshot. Texas and Snowball both filed a finding its own
+ * screenshot contradicted.
+ *
+ * It goes to `persona_playtest.act`, the run's one step that accepts an image
+ * (the report step accepts none), with no tools and a one word ceiling, and
+ * it is spent from the same run budget the warning at 80 percent keeps for the
+ * report step. A budget refusal, or any other refusal, ends the checks and
+ * keeps the report as it is.
+ *
+ * @returns {Promise<{report: Object, downgraded: Array<Object>}>}
+ */
+export async function checkClaimsAgainstScreenshots({ client, jobId, runId, report, screenshots, runDirectory, ledger, transcript, log = () => {} }) {
+  if (!report || !Array.isArray(report.findings)) return { report, downgraded: [] };
+  const targets = report.findings.filter(finding => finding && ABOVE_MINOR.includes(finding.severity)).slice(0, MAX_CLAIM_CHECKS);
+  const downgraded = [];
+  const verdicts = new Map();
+  for (const finding of targets) {
+    const shot = citedAfterScreenshot(finding, screenshots);
+    if (!shot) {
+      transcript.write({ kind: 'claim_check', id: finding.id, screenshot: null, answer: 'skipped', reason: 'no cited screenshot' });
+      continue;
+    }
+    let data;
+    try {
+      data = (await readFile(path.join(runDirectory, shot.relativePath))).toString('base64');
+    } catch {
+      transcript.write({ kind: 'claim_check', id: finding.id, screenshot: shot.relativePath, answer: 'skipped', reason: 'the screenshot could not be read' });
+      continue;
+    }
+    let answer;
+    const askedAt = Date.now();
+    try {
+      answer = await callStepWithRetry({
+        client,
+        jobId,
+        runId,
+        step: STEPS.personaAct,
+        system: CLAIM_CHECK_SYSTEM,
+        messages: [{
+          role: 'user',
+          content: [
+            { type: 'image', source: { type: 'base64', media_type: 'image/png', data } },
+            { type: 'text', text: `Claim (${finding.severity}): ${finding.title || ''}\n${String(finding.description || '').slice(0, 1200)}\n\nDoes this image contradict the claim: yes, no, unclear?` }
+          ]
+        }],
+        maxTokens: CLAIM_CHECK_MAX_TOKENS,
+        escalate: false,
+        maxAttempts: 1
+      });
+    } catch (error) {
+      const reason = error instanceof ProxyError && error.isBudget ? 'the run budget is spent' : String(error && error.message ? error.message : error);
+      transcript.write({ kind: 'claim_check', id: finding.id, screenshot: shot.relativePath, answer: 'skipped', reason });
+      log(`the screenshot check stopped (${reason}); the report is kept as written.`);
+      break;
+    }
+    if (ledger) ledger.record({ model: answer.model, usage: answer.usage, ms: Date.now() - askedAt });
+    const verdict = claimCheckAnswer(answer.text);
+    transcript.write({ kind: 'claim_check', id: finding.id, screenshot: shot.relativePath, answer: verdict });
+    if (verdict === 'yes' && DOWNGRADE[finding.severity]) verdicts.set(finding, shot.relativePath);
+  }
+  if (verdicts.size === 0) return { report, downgraded };
+  const findings = report.findings.map(finding => {
+    if (!verdicts.has(finding)) return finding;
+    const screenshot = verdicts.get(finding);
+    const to = DOWNGRADE[finding.severity];
+    const note = `its own screenshot ${screenshot} contradicts the claim`;
+    downgraded.push({ id: finding.id, from: finding.severity, to, screenshot, note });
+    const description = String(finding.description || '');
+    return { ...finding, severity: to, description: description ? `${description}\n\nDowngraded by the runner: ${note}.` : `Downgraded by the runner: ${note}.` };
+  });
+  return { report: { ...report, findings }, downgraded };
+}
+
 /** The smallest output ceiling worth retrying a report turn with. */
 export const REPORT_MIN_TOKENS = 1536;
 
@@ -1024,8 +1212,9 @@ async function callWithImageRetry({ imageBudget, keepObservations, messages, onI
  * product where the same few thousand tokens are re-sent tens of times. The
  * proxy allowlists `system` and passes the blocks through unchanged.
  */
-export function buildSystem({ skill, persona, brief, intentNotes, driverName, target, maxActions, runId = null }) {
+export function buildSystem({ skill, persona, brief, localBrief = {}, intentNotes, driverName, target, maxActions, runId = null }) {
   const identity = runIdentity({ persona, runId });
+  const account = testAccountOf(localBrief);
   return [
     { type: 'text', text: skill },
     { type: 'text', text: `<persona_file>\n${persona.body}\n</persona_file>` },
@@ -1037,7 +1226,9 @@ export function buildSystem({ skill, persona, brief, intentNotes, driverName, ta
         `max_actions: ${maxActions}`,
         `<expectations_brief>\n${briefForPrompt(brief)}\n</expectations_brief>`,
         `<design_intent_notes>\n${intentNotesForPrompt(brief, intentNotes)}\n</design_intent_notes>`,
-        ...(identity ? [`<run_identity>\n${runIdentityForPrompt(identity)}\n</run_identity>`] : [])
+        `<expected_noise>\nConsole errors and failed requests matching these are expected and never a finding:\n${expectedNoiseForPrompt(localBrief && localBrief.expected_noise)}\n</expected_noise>`,
+        ...(account ? [`<test_account>\n${testAccountForPrompt(account)}\n</test_account>`] : []),
+        ...(identity ? [`<run_identity>\n${runIdentityForPrompt(identity, { testAccount: Boolean(account) })}\n</run_identity>`] : [])
       ].join('\n'),
       cache_control: { type: 'ephemeral' }
     }
@@ -1077,11 +1268,41 @@ export function runIdentity({ persona, runId }) {
  * @param {{email: string, displayName: string}} identity
  * @returns {string}
  */
-export function runIdentityForPrompt(identity) {
+export function runIdentityForPrompt(identity, { testAccount = false } = {}) {
   return [
     `email: ${identity.email}`,
     `display_name: ${identity.displayName}`,
-    'Use this email and this display name whenever a form asks for an email, a name, a username or a coach or player name, unless the brief says otherwise. They are new for this run, so an account made with them does not exist yet. Never reuse an email from an earlier run.'
+    testAccount
+      ? 'Sign in with the test account above. Use this email and this display name only if you have to make a new account, and whenever a form asks for a display name, a coach or a player name. Never reuse an email from an earlier run.'
+      : 'Use this email and this display name whenever a form asks for an email, a name, a username or a coach or player name, unless the brief says otherwise. They are new for this run, so an account made with them does not exist yet. Never reuse an email from an earlier run.'
+  ].join('\n');
+}
+
+/**
+ * The local brief's test account, or null when it has none that can be used.
+ *
+ * @param {Object} localBrief
+ * @returns {{email: string, password: string, how_to_sign_in: string|null}|null}
+ */
+export function testAccountOf(localBrief) {
+  const account = localBrief && localBrief.test_account;
+  if (!account || typeof account !== 'object' || typeof account.email !== 'string' || !account.email || typeof account.password !== 'string' || account.password.length < MIN_PASSWORD_CHARS) return null;
+  return { email: account.email, password: account.password, how_to_sign_in: typeof account.how_to_sign_in === 'string' && account.how_to_sign_in.trim() ? account.how_to_sign_in.trim() : null };
+}
+
+/**
+ * The test account block of the persona prompt: an account the developer made
+ * so a run can start past sign up.
+ *
+ * @param {{email: string, password: string, how_to_sign_in: string|null}} account
+ * @returns {string}
+ */
+export function testAccountForPrompt(account) {
+  return [
+    `email: ${account.email}`,
+    `password: ${account.password}`,
+    ...(account.how_to_sign_in ? [`how_to_sign_in: ${account.how_to_sign_in}`] : []),
+    'This is a throwaway account the developer made for this playtest. Sign in with it rather than signing up, so your actions go to the game itself. Never write the password in a note or a report.'
   ].join('\n');
 }
 
@@ -1152,11 +1373,54 @@ export function assistantContent(answer) {
   return content;
 }
 
-/** Keep the transcript readable without storing a whole screenshot in it. */
-function redactToolInput(toolUse) {
+/** What a typed secret is written as in the transcript and the action log. */
+export const REDACTED_SECRET = '[test password]';
+
+/**
+ * Every string in a value with each secret replaced by {@link REDACTED_SECRET},
+ * through arrays and plain objects. A new value; the input is not changed.
+ *
+ * @param {any} value
+ * @param {Array<string>} secrets
+ * @returns {any}
+ */
+export function scrubSecrets(value, secrets = []) {
+  if (!Array.isArray(secrets) || secrets.length === 0) return value;
+  if (typeof value === 'string') {
+    let text = value;
+    // Whole tokens only: the password with no letter, digit or underscore
+    // directly before or after it, so it is replaced where it was written
+    // (alone, quoted, after a colon) and never inside a longer word.
+    for (const secret of secrets) {
+      if (!secret) continue;
+      const escaped = secret.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+      text = text.replace(new RegExp(`(?<![\\p{L}\\p{N}_])${escaped}(?![\\p{L}\\p{N}_])`, 'gu'), REDACTED_SECRET);
+    }
+    return text;
+  }
+  if (Array.isArray(value)) return value.map(entry => scrubSecrets(entry, secrets));
+  if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, scrubSecrets(entry, secrets)]));
+  return value;
+}
+
+/**
+ * The strings the transcript must not repeat: the local brief's test account
+ * password. It is in the prompt (that is how a persona can type it), but the
+ * transcript can be uploaded and the action log is shown in the report step.
+ *
+ * @param {Object} localBrief
+ * @returns {Array<string>}
+ */
+export function secretsOf(localBrief) {
+  const password = localBrief && localBrief.test_account && localBrief.test_account.password;
+  return typeof password === 'string' && password.length >= MIN_PASSWORD_CHARS ? [password] : [];
+}
+
+/** Keep the transcript readable without storing a whole screenshot in it, or a test account password. */
+export function redactToolInput(toolUse, secrets = []) {
   if (!toolUse.input) return null;
   const { thought_in_character: ignored, ...rest } = toolUse.input;
-  return rest;
+  return scrubSecrets(rest, secrets);
 }
 
 async function safeScreenshot(driver, name, screenshots, transcript, observation = null) {
@@ -1299,6 +1563,8 @@ export function notesToFindings(notes, screenshots) {
     if (!category || !note.text) continue;
     // A blocker or major written on a driver error is the runner's failure.
     if (note.driver_error && ABOVE_MINOR.includes(note.severity)) continue;
+    // Console noise the developer said to expect is never a finding.
+    if (note.expected_noise) continue;
     const text = String(note.text);
     const shot = nearestScreenshot(note, screenshots);
     findings.push({
@@ -1377,6 +1643,7 @@ export function noteLine(note, i) {
   const tags = [
     note.automatic ? 'automatic' : null,
     note.driver_error ? 'written on a driver error' : null,
+    note.expected_noise ? 'expected noise' : null,
     Number.isInteger(note.observation) && note.observation > 0 ? `seen at observation ${note.observation}` : null,
     note.screenshot ? `screenshot ${note.screenshot}` : null
   ].filter(Boolean);
@@ -1449,8 +1716,8 @@ export function observationTitle(observation) {
  */
 /** How report.md lists what the runner dropped. */
 export const RUNNER_DROPPED_SECTION = Object.freeze({
-  heading: '## Dropped by the runner (driver errors or waiting notes only)',
-  explanation: 'The runner removes a blocker or major whose only cited observations are driver errors (the runner could not carry out an action; the game did nothing wrong), and any finding cited only by waiting notes. They are not sent to the server. They are kept here so you can judge them yourself.',
+  heading: '## Dropped by the runner (driver errors, waiting notes or expected noise only)',
+  explanation: 'The runner removes a blocker or major whose only cited observations are driver errors (the runner could not carry out an action; the game did nothing wrong), and any finding cited only by waiting notes or by console noise the brief says to expect. They are not sent to the server. They are kept here so you can judge them yourself.',
   emptyNote: '- Nothing here that the runner kept. See the dropped findings below.'
 });
 
@@ -1473,11 +1740,26 @@ export const RUNNER_DROPPED_SECTION = Object.freeze({
 export function dropRunnerRejectedFindings(report, { notes = [], observations = [] } = {}) {
   if (!report || !Array.isArray(report.findings)) return { report, dropped: [] };
   const byIndex = new Map(observations.map(entry => [entry.index, entry]));
+  // The runner's console-error screenshot of a moment whose every console
+  // note was expected noise says nothing more than those notes do.
+  const noiseShots = new Set();
+  const shotNotes = new Map();
+  for (const note of notes) {
+    if (!note || !note.automatic || !note.screenshot) continue;
+    if (!shotNotes.has(note.screenshot)) shotNotes.set(note.screenshot, []);
+    shotNotes.get(note.screenshot).push(note);
+  }
+  for (const [shot, attached] of shotNotes) if (attached.every(note => note.expected_noise)) noiseShots.add(shot);
   const verdict = finding => {
     const cited = [];
     for (const item of Array.isArray(finding.evidence) ? finding.evidence : []) {
-      if (!item || item.type !== 'transcript_step') continue;
+      if (!item) continue;
       const ref = String(item.ref || '');
+      if (item.type === 'screenshot') {
+        if (noiseShots.has(ref)) cited.push('noise');
+        continue;
+      }
+      if (item.type !== 'transcript_step') continue;
       const observation = ref.match(/^\s*observation\s*#?\s*(\d+)\s*$/i);
       const note = ref.match(/^\s*note\s*#?\s*(\d+)\s*$/i);
       if (observation) {
@@ -1485,7 +1767,7 @@ export function dropRunnerRejectedFindings(report, { notes = [], observations =
         cited.push(entry && entry.driver_error ? 'driver' : 'other');
       } else if (note) {
         const entry = notes[Number(note[1]) - 1];
-        cited.push(entry && entry.kind === 'waiting' ? 'waiting' : entry && entry.driver_error ? 'driver' : 'other');
+        cited.push(entry && entry.kind === 'waiting' ? 'waiting' : entry && entry.expected_noise ? 'noise' : entry && entry.driver_error ? 'driver' : 'other');
       } else {
         cited.push('other');
       }
@@ -1493,7 +1775,9 @@ export function dropRunnerRejectedFindings(report, { notes = [], observations =
     if (cited.length === 0) return null;
     if (ABOVE_MINOR.includes(finding.severity) && cited.every(kind => kind === 'driver')) return 'driver errors only';
     if (cited.every(kind => kind === 'waiting')) return 'waiting notes only';
-    if (ABOVE_MINOR.includes(finding.severity) && cited.every(kind => kind === 'driver' || kind === 'waiting')) return 'driver errors and waiting notes only';
+    if (cited.every(kind => kind === 'noise')) return 'expected console noise only';
+    if (cited.every(kind => kind === 'waiting' || kind === 'noise')) return 'expected console noise and waiting notes only';
+    if (ABOVE_MINOR.includes(finding.severity) && cited.every(kind => kind === 'driver' || kind === 'waiting' || kind === 'noise')) return 'driver errors, waiting notes and expected noise only';
     return null;
   };
   const dropped = [];
@@ -1722,7 +2006,7 @@ export function downgradeContradictedFindings(report, log) {
 }
 
 /** How report.md lists the findings the runner downgraded. */
-export const DOWNGRADED_HEADING = '## Downgraded by the runner (the action log contradicts the claim)';
+export const DOWNGRADED_HEADING = '## Downgraded by the runner (the action log or its own screenshot contradicts the claim)';
 
 /**
  * report.md agreeing with report.json about downgraded findings: on every