ravensight-playtest 0.2.6 → 0.2.8

package/src/run/personaLoop.js

@@ -11,6 +11,7 @@ import { assertNoSecrets } from './secretScan.js';
 import { ACTION_TOOLS, PLAY_TOOLS, SWIPE_TOOL, REPORT_TOOLS, executeTool } from './tools.js';
 import { createTranscript } from './transcript.js';
 import { createUsageLedger, writeUsageFile } from './usage.js';
+import { fillPlaceholders, fillReportPlaceholders } from './placeholders.js';
 
 /**
  * One persona's playthrough: PLAYING then REPORTING (spec 03).
@@ -51,8 +52,22 @@ const nextOrder = () => {
   return ORDER;
 };
 
-/** `Failed to load resource: ... status of 4xx`: a missing asset or a refused call, not a crash. */
-const RESOURCE_4XX = /Failed to load resource: the server responded with a status of 4\d\d/;
+/**
+ * A 4xx on a request, in the shapes a browser console and a game's own logging
+ * print it: `Failed to load resource: ... status of 4xx`, `HTTP 404`,
+ * `status: 401`, `statusCode=403`, `401 Unauthorized`. A missing asset or a
+ * refused call, not a crash.
+ */
+export const RESOURCE_4XX = /Failed to load resource: the server responded with a status of 4\d\d|\bHTTP 4\d\d\b|status(?:Code)?\s*[:=]?\s*4\d\d\b|\b4\d\d (?:Unauthorized|Forbidden|Not Found|Bad Request)\b/i;
+
+/** The screenshots the runner takes on its own, which no persona described. */
+const RUNNER_SCREENSHOTS = Object.freeze(['landing', 'console-error', 'final']);
+
+/** Severities above `minor`, which network notes alone cannot earn. */
+const ABOVE_MINOR = Object.freeze(['blocker', 'major']);
+
+/** What a capped finding's description gains, so the reader knows why. */
+export const NETWORK_CAP_NOTE = 'Severity capped at minor by the runner: the only evidence is automatic network notes, and a failed request is a finding above minor only when the persona saw the game break because of it.';
 
 /** A persona note kind as a report finding category. */
 export const NOTE_CATEGORY = Object.freeze({ bug: 'bug', confusing: 'ux', suggestion: 'ux', good: 'ux', reaction: 'ux' });
@@ -127,6 +142,10 @@ export async function runPersona(args) {
   const startedAt = new Date().toISOString();
   const notes = [];
   const screenshots = [];
+  // Every observation the persona was shown, by index, so a note can carry
+  // what the screen looked like next and a finding can cite where a failure
+  // is visible.
+  const observations = [];
   let imageBudget = maxImages;
 
   await mkdir(path.join(runDirectory, 'screenshots'), { recursive: true });
@@ -186,6 +205,7 @@ export async function runPersona(args) {
       ledger,
       notes,
       screenshots,
+      observations,
       imageBudget,
       keepObservations,
       onImageBudget: next => {
@@ -242,6 +262,7 @@ export async function runPersona(args) {
       ledger,
       notes,
       screenshots,
+      observations,
       play,
       startedAt,
       endedAt,
@@ -305,6 +326,7 @@ async function playLoop({
   ledger,
   notes,
   screenshots,
+  observations = [],
   imageBudget,
   keepObservations,
   heartbeat,
@@ -343,7 +365,26 @@ async function playLoop({
   let wrapUpLeft = Infinity;
   const maxTurns = maxActions * 3 + 20;
 
+  // Notes waiting for the next observation, which is the screen they are
+  // checked against: a failure the next screen does not show is not a
+  // blocker.
+  let awaitingAfter = [];
+  const observed = observation => {
+    const entry = { index: observations.length + 1, step: observation.step, url: observation.url || null, title: observationTitle(observation) };
+    observations.push(entry);
+    for (const note of awaitingAfter) note.after = { url: entry.url, title: entry.title };
+    awaitingAfter = [];
+    return entry;
+  };
+  const noted = note => {
+    const entry = { ...note, at_action: actions, observation: observations.length, order: nextOrder() };
+    notes.push(entry);
+    awaitingAfter.push(entry);
+    return entry;
+  };
+
   const first = await driver.observe();
+  observed(first);
   latestObservation = first.text;
   visualObservation = Boolean(first.screenshotBase64);
   transcript.write({ kind: 'observation', step: first.step, text: first.text, url: first.url });
@@ -458,7 +499,7 @@ async function playLoop({
           // (bug, confusing, good, suggestion, reaction) would otherwise
           // overwrite the transcript line's `kind`, and every note would
           // read as a different sort of line.
-          notes.push({ ...note, at_action: actions, order: nextOrder() });
+          noted(note);
           transcript.write({ kind: 'note', tool: toolUse.name, note });
         },
         onScreenshot: shot => {
@@ -479,6 +520,7 @@ async function playLoop({
         actions
       });
       if (executed.observation) {
+        observed(executed.observation);
         latestObservation = executed.observation.text;
         visualObservation = Boolean(executed.observation.screenshotBase64);
         recentActions.push(`${toolUse.name}: ${executed.changed ? 'screen changed' : 'no observed change'}`);
@@ -513,8 +555,8 @@ async function playLoop({
       const shot = await safeScreenshot(driver, 'console-error', screenshots, transcript);
       for (const entry of consoleErrors) {
         const kind = RESOURCE_4XX.test(String(entry.text)) ? 'network' : 'bug';
-        const note = { kind, text: `${entry.type}: ${entry.text}`, automatic: true };
-        notes.push({ ...note, at_action: actions, order: nextOrder() });
+        const note = { kind, text: `${entry.type}: ${entry.text}`, automatic: true, ...(shot ? { screenshot: shot.relativePath } : {}) };
+        noted(note);
         transcript.write({ kind: 'note', note });
       }
       asides.push(`The game logged ${consoleErrors.length} error(s): ${consoleErrors.map(entry => entry.text).join(' | ').slice(0, 800)}${shot ? ` A screenshot is saved at ${shot.relativePath}.` : ''}`);
@@ -584,6 +626,7 @@ async function reportPhase({
   ledger,
   notes,
   screenshots,
+  observations = [],
   play,
   startedAt,
   endedAt,
@@ -600,9 +643,17 @@ async function reportPhase({
         `actions taken: ${play.actionsTaken}   stop reason: ${play.quitReason}${play.quitDetail ? ` (${play.quitDetail})` : ''}`,
         '',
         'Notes you logged, in order:',
-        notes.length > 0 ? notes.map((note, i) => `${i + 1}. [${note.kind}${note.severity ? `/${note.severity}` : ''}] ${note.text}`).join('\n') : '(none)',
+        notes.length > 0 ? notes.map((note, i) => noteLine(note, i)).join('\n') : '(none)',
         '',
-        'Notes marked automatic were recorded by the runner from the console, with no severity: decide one from what the persona saw. Network notes are context, not bugs: a failed request is a finding only when the persona saw a broken screen because of it.',
+        'Notes marked automatic were recorded by the runner from the console, with no severity: decide one from what the persona saw. Network notes are context, not bugs: a failed request is a finding only when the persona saw a broken screen because of it. A finding whose only evidence is automatic network notes is capped at minor by the runner.',
+        '',
+        'Observations the persona was shown, by index:',
+        observationLines(observations),
+        '',
+        'Each note carries "after": the URL and title of the observation that followed it. A blocker or major finding must cite the observation index where the failure is visible, as evidence {"type": "transcript_step", "ref": "observation N"}, and must not be contradicted by its note\'s "after" state or by a later persona note: if the next screen or a later note shows the game working, it is not a blocker or a major. Cite a note as {"type": "transcript_step", "ref": "note N"}.',
+        '',
+        'Never write calendar dates or times in the prose of report.md or report.json. The runner stamps when the report was generated, so leave the template\'s Date out rather than writing a placeholder for it.',
+        'Replace every <...> slot in the template with real content. Never copy a <...> placeholder into the report.',
         '',
         'Screenshots available as evidence (use these exact paths in evidence refs):',
         screenshots.length > 0 ? screenshots.map(shot => `- ${shot.relativePath}${shot.reason ? ` (${shot.reason})` : ''}`).join('\n') : '(none)',
@@ -669,7 +720,7 @@ async function reportPhase({
       quitReason: play.quitReason,
       quitDetail: play.quitDetail
     };
-    const candidate = parsed ? stampReport(parsed, facts) : null;
+    const candidate = parsed ? capNetworkOnlyFindings(stampReport(parsed, facts), { notes, screenshots }) : null;
     const validation = candidate ? validator.validate(candidate) : { ok: false, details: [{ instancePath: '/', message: parseError || 'report.json was not written' }] };
     const evidenceProblems = candidate ? await checkEvidence(candidate, runDirectory) : [];
     // The server lints the same four sections at `complete` and records the
@@ -715,6 +766,11 @@ async function reportPhase({
   if (!markdown) {
     markdown = renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason: 'The report step wrote report.json but no report.md.' });
   }
+  const generatedAt = new Date().toISOString();
+  const facts = reportFacts({ persona, driverName, target, play });
+  markdown = stampGeneratedAt(markdown, generatedAt, facts);
+  // The same slots in the prose the dashboard shows from report.json.
+  report = fillReportPlaceholders(report, { generatedAt, ...facts });
 
   await writeReportFiles({ runDirectory, markdown, report });
   return { report, markdown, lint };
@@ -1033,17 +1089,167 @@ async function writePartialReport({ outcome, runDirectory, persona, driverName,
     play,
     reason: 'The run reached its model budget before the report step. What the persona saw is below, and report.json carries the notes as unreviewed findings.'
   });
-  await writeReportFiles({ runDirectory, markdown, report });
+  const stamped = stampGeneratedAt(markdown, new Date().toISOString(), reportFacts({ persona, driverName, target, play }));
+  await writeReportFiles({ runDirectory, markdown: stamped, report });
   outcome.report = report;
-  outcome.reportMarkdown = markdown;
+  outcome.reportMarkdown = stamped;
 }
 
 async function writePartialMarkdown({ runDirectory, persona, driverName, target, notes, screenshots, play, reason }) {
-  const markdown = renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason });
+  const markdown = stampGeneratedAt(renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason }), new Date().toISOString(), reportFacts({ persona, driverName, target, play }));
   assertNoSecrets('report.md', markdown);
   await writeFile(path.join(runDirectory, RUN_FILES.reportMd), markdown, 'utf8');
 }
 
+/**
+ * A note as one numbered line of the report prompt: its kind, whether the
+ * runner recorded it, the observation it was made on and the screen that
+ * followed it.
+ *
+ * @param {Object} note
+ * @param {number} i
+ * @returns {string}
+ */
+export function noteLine(note, i) {
+  const tags = [
+    note.automatic ? 'automatic' : null,
+    Number.isInteger(note.observation) && note.observation > 0 ? `seen at observation ${note.observation}` : null,
+    note.screenshot ? `screenshot ${note.screenshot}` : null
+  ].filter(Boolean);
+  const after = note.after
+    ? ` after: ${JSON.stringify({ url: note.after.url ?? null, title: note.after.title ?? null })}`
+    : ' after: (no later observation)';
+  return `${i + 1}. [${note.kind}${note.severity ? `/${note.severity}` : ''}] ${note.text}${tags.length > 0 ? ` (${tags.join(', ')})` : ''}${after}`;
+}
+
+/** The most observations listed in the report prompt; the newest are kept. */
+export const MAX_OBSERVATIONS_IN_REPORT = 200;
+
+function observationLines(observations) {
+  if (!observations || observations.length === 0) return '(none)';
+  const shown = observations.slice(-MAX_OBSERVATIONS_IN_REPORT);
+  const lines = shown.map(entry => `observation ${entry.index}: step ${entry.step}${entry.url ? ` ${entry.url}` : ''}${entry.title ? ` "${entry.title}"` : ''}`);
+  if (shown.length < observations.length) lines.unshift(`(the first ${observations.length - shown.length} are left out)`);
+  return lines.join('\n');
+}
+
+/**
+ * A short title for an observation: the page title when the driver has one,
+ * otherwise the first line of what was observed.
+ *
+ * @param {Object} observation
+ * @returns {string|null}
+ */
+export function observationTitle(observation) {
+  if (observation && typeof observation.title === 'string' && observation.title.trim()) return observation.title.trim().slice(0, 120);
+  const line = String((observation && observation.text) || '').split('\n').map(part => part.trim()).find(Boolean);
+  return line ? line.slice(0, 80) : null;
+}
+
+/**
+ * Cap at `minor` every finding whose evidence is only automatic network
+ * notes, in code rather than by asking.
+ *
+ * A finding rests only on network notes when it cites at least one (a
+ * `note N` ref to an automatic network note, or the runner's console-error
+ * screenshot whose notes were all network notes) and nothing else: no persona
+ * note, no automatic non-network note, no observation ref, no screenshot the
+ * persona took, no other evidence type. Only the runner's own landing and
+ * final screenshots neither count for nor against it.
+ *
+ * @param {Object} report a stamped report
+ * @param {{notes: Array<Object>, screenshots: Array<Object>}} context
+ * @returns {Object} the report, with capped findings rewritten
+ */
+export function capNetworkOnlyFindings(report, { notes = [], screenshots = [] } = {}) {
+  if (!report || !Array.isArray(report.findings)) return report;
+  const byShot = new Map();
+  for (const note of notes) {
+    if (!note || !note.automatic || !note.screenshot) continue;
+    if (!byShot.has(note.screenshot)) byShot.set(note.screenshot, []);
+    byShot.get(note.screenshot).push(note);
+  }
+  const reasons = new Map(screenshots.filter(shot => shot && shot.relativePath).map(shot => [shot.relativePath, String(shot.reason || '')]));
+
+  const verdict = finding => {
+    let network = 0;
+    for (const item of Array.isArray(finding.evidence) ? finding.evidence : []) {
+      if (!item) continue;
+      const ref = String(item.ref || '');
+      if (item.type === 'transcript_step') {
+        // An observation ref, or anything that is not a network note, is
+        // other evidence.
+        const match = ref.match(/^\s*note\s*#?\s*(\d+)\s*$/i);
+        const note = match ? notes[Number(match[1]) - 1] : null;
+        if (note && note.automatic && note.kind === 'network') network += 1;
+        else return false;
+      } else if (item.type === 'screenshot') {
+        const attached = byShot.get(ref);
+        if (attached && attached.length > 0) {
+          if (attached.every(note => note.kind === 'network')) network += 1;
+          else return false;
+          continue;
+        }
+        // The runner's own landing and final screenshots say nothing either
+        // way; any screenshot the persona took is other evidence.
+        const reason = reasons.has(ref) ? reasons.get(ref) : null;
+        if (reason !== null && RUNNER_SCREENSHOTS.includes(reason)) continue;
+        return false;
+      } else {
+        return false;
+      }
+    }
+    return network > 0;
+  };
+
+  return {
+    ...report,
+    findings: report.findings.map(finding => {
+      if (!finding || !ABOVE_MINOR.includes(finding.severity) || !verdict(finding)) return finding;
+      const description = String(finding.description || '');
+      return { ...finding, severity: 'minor', description: description ? `${description}\n\n${NETWORK_CAP_NOTE}` : NETWORK_CAP_NOTE };
+    })
+  };
+}
+
+/**
+ * Put the runner's own timestamp at the head of a report.md, right under
+ * its first heading, replacing one stamped before. Whatever the model wrote
+ * in its prose is left alone; the header is what says when the report was
+ * generated.
+ *
+ * @param {string} markdown
+ * @param {string} generatedAt an ISO timestamp
+ * @returns {string}
+ */
+export function stampGeneratedAt(markdown, generatedAt, facts = {}) {
+  const line = `**Generated at:** ${generatedAt}`;
+  const lines = String(markdown || '').split('\n').filter(entry => !/^\*\*Generated at:\*\* /.test(entry));
+  const heading = lines.findIndex(entry => /^# /.test(entry));
+  if (heading === -1) lines.unshift(line, '');
+  else lines.splice(heading + 1, 0, line);
+  // The template's own slots (`**Date:** <date>`) are filled from what the
+  // runner knows, because the prompt forbids the model a calendar date and a
+  // model that obeys both writes the placeholder itself.
+  return fillPlaceholders(lines.join('\n'), { generatedAt, ...facts });
+}
+
+/**
+ * What the runner knows about a run, for the template slots a report left
+ * unfilled.
+ *
+ * @returns {{driver: string, target: string, persona: string, actionsTaken: number, quitReason: string}}
+ */
+function reportFacts({ persona, driverName, target, play }) {
+  return {
+    driver: driverName,
+    target,
+    persona: persona ? persona.name || persona.slug : undefined,
+    actionsTaken: play ? play.actionsTaken : undefined,
+    quitReason: play ? play.quitReason : undefined
+  };
+}
+
 /**
  * The `prompt_version` the report has to carry, read out of the skill's own
  * frontmatter so a content pack update moves it without a CLI release.
@@ -1056,4 +1262,4 @@ export function promptVersionOf(skill) {
   return match ? match[1] : null;
 }
 
-export default { runPersona, buildSystem, promptVersionOf, shouldEscalate, renderFallbackMarkdown, notesToFindings, nearestScreenshot };
+export default { runPersona, buildSystem, promptVersionOf, shouldEscalate, renderFallbackMarkdown, notesToFindings, nearestScreenshot, capNetworkOnlyFindings, stampGeneratedAt, noteLine, observationTitle };

package/src/run/placeholders.js

@@ -0,0 +1,186 @@
+/**
+ * Template placeholders a model copied into a report instead of filling.
+ *
+ * The persona skill's report template reads `**Date:** <date> ...`, and the
+ * report prompt says never to write a calendar date, so a model that obeys
+ * both writes the placeholder itself (`<run date>`) and the developer reads
+ * it as the report's date.
+ *
+ * Deliberately narrow, because angle brackets in a report are usually real:
+ * a key (`<Enter>`, `<Ctrl+A>`), an element (`<form>`, `<title>`), a type
+ * (`Array<string>`), a link, or the game's own text. So only two things are
+ * treated as a slot:
+ *
+ * 1. A slot the runner knows (date, driver, target, persona, actions, quit
+ *    reason, game, version), filled from its own facts, or "not recorded"
+ *    when it has no value for it.
+ * 2. Lowercase words in brackets on a `**Field:**` line, the shape of the
+ *    template's header, which become "not recorded".
+ *
+ * Never touched: a `<` straight after a word character, a key name, anything
+ * quoted, HTML, autolinks, code spans and fenced blocks.
+ */
+
+/** What a slot with no value becomes. */
+export const NOT_RECORDED = 'not recorded';
+
+const HTML_TAGS = new Set([
+  'a', 'abbr', 'article', 'aside', 'b', 'blockquote', 'body', 'br', 'button', 'canvas', 'caption', 'code', 'dd', 'del',
+  'details', 'dialog', 'div', 'dl', 'dt', 'em', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2', 'h3', 'h4', 'h5',
+  'h6', 'head', 'header', 'hr', 'html', 'i', 'iframe', 'img', 'input', 'ins', 'kbd', 'label', 'li', 'link', 'main', 'mark',
+  'meta', 'nav', 'ol', 'option', 'p', 'picture', 'pre', 'q', 's', 'samp', 'script', 'section', 'select', 'small',
+  'source', 'span', 'strong', 'style', 'sub', 'summary', 'sup', 'table', 'tbody', 'td', 'template', 'textarea',
+  'tfoot', 'th', 'thead', 'title', 'tr', 'u', 'ul', 'video'
+]);
+
+const KEY_NAMES = new Set([
+  'enter', 'return', 'tab', 'space', 'spacebar', 'esc', 'escape', 'backspace', 'delete', 'del', 'insert', 'home', 'end',
+  'pageup', 'pagedown', 'up', 'down', 'left', 'right', 'arrowup', 'arrowdown', 'arrowleft', 'arrowright', 'shift',
+  'ctrl', 'control', 'alt', 'option', 'cmd', 'command', 'meta', 'super', 'win', 'fn', 'capslock'
+]);
+
+/** `<...>` with no nested bracket and no line break, short enough to be a template slot. */
+const ANGLE = /<([^<>\n]{1,80})>/g;
+
+/** A line in the template's header shape: `**Date:** ...`. */
+const FIELD_LINE = /\*\*[^*\n]+:\*\*/;
+
+const QUOTES = { '"': '"', "'": "'", '“': '”', '‘': '’', '`': '`' };
+
+function normalise(inner) {
+  return String(inner).trim().toLowerCase().replace(/[_-]+/g, ' ').replace(/\s+/g, ' ');
+}
+
+/**
+ * Whether the bracket is a slot the runner knows, and its value if it has one.
+ *
+ * @param {string} inner
+ * @param {Object} facts
+ * @returns {{known: boolean, value: string|null}}
+ */
+function knownSlot(inner, facts) {
+  const key = normalise(inner);
+  const date = facts.generatedAt ? String(facts.generatedAt).slice(0, 10) : null;
+  const actions = facts.actionsTaken === null || facts.actionsTaken === undefined || !Number.isFinite(Number(facts.actionsTaken))
+    ? null
+    : Number(facts.actionsTaken);
+  const slot = value => ({ known: true, value: value === null || value === undefined || value === '' ? null : String(value) });
+  if (/^(run |report |session )?(date|timestamp|date and time)$/.test(key) || key === 'generated at') return slot(date);
+  if (/^driver( name)?$/.test(key)) return slot(facts.driver);
+  if (/^(target|target url|build url|game url)$/.test(key)) return slot(facts.target);
+  if (/^persona( name| slug)?$/.test(key)) return slot(facts.persona);
+  if (/^(n|number of) actions$/.test(key)) return slot(actions === null ? null : `${actions} actions`);
+  if (key === 'actions taken') return slot(actions);
+  if (/^(quit|stop) reason$/.test(key)) return slot(facts.quitReason);
+  if (key === 'game' || key === 'game name') return slot(facts.game);
+  if (key === 'version' || key === 'game version') return slot(facts.version);
+  return { known: false, value: null };
+}
+
+/**
+ * The replacement for one bracket match, or null to leave it as written.
+ *
+ * @param {string} text the prose segment the match is in
+ * @param {string} whole the match, brackets included
+ * @param {string} inner
+ * @param {number} offset
+ * @param {string} line the whole line, for the field check
+ * @param {Object} facts
+ * @returns {string|null}
+ */
+function replacementFor(text, whole, inner, offset, line, facts) {
+  const before = text[offset - 1];
+  const after = text[offset + whole.length];
+  // `Array<string>`, `Map<K, V>`: a type, not a slot.
+  if (before !== undefined && /\w/.test(before)) return null;
+  // The game's own words, quoted as evidence.
+  if (before !== undefined && QUOTES[before] !== undefined && QUOTES[before] === after) return null;
+  const value = String(inner).trim();
+  if (!/^[A-Za-z]/.test(value)) return null;
+  // A key or a chord: <Enter>, <Ctrl+A>, <Shift + Tab>.
+  if (value.includes('+') || KEY_NAMES.has(value.toLowerCase())) return null;
+  const known = knownSlot(value, facts);
+  if (known.known) return known.value || NOT_RECORDED;
+  if (FIELD_LINE.test(line) && /^[a-z]+(?: [a-z]+)*$/.test(value) && !HTML_TAGS.has(value)) return NOT_RECORDED;
+  return null;
+}
+
+/**
+ * Walk the prose of a markdown document (not fenced blocks, not code spans),
+ * handing each segment and its whole line to `fn`.
+ *
+ * @param {string} markdown
+ * @param {(text: string, line: string) => string} fn
+ * @returns {string}
+ */
+function mapProse(markdown, fn) {
+  let fenced = false;
+  return String(markdown == null ? '' : markdown).split('\n').map(line => {
+    if (/^\s*(```|~~~)/.test(line)) {
+      fenced = !fenced;
+      return line;
+    }
+    if (fenced) return line;
+    return line.split(/(`[^`]*`)/).map(part => (part.length > 1 && part.startsWith('`') && part.endsWith('`') ? part : fn(part, line))).join('');
+  }).join('\n');
+}
+
+/**
+ * Fill the slots the runner knows, and the header fields it does not, from
+ * its own facts. Everything else in angle brackets is left as written.
+ *
+ * @param {string} markdown
+ * @param {{generatedAt?: string, driver?: string, target?: string, persona?: string,
+ *   actionsTaken?: number, quitReason?: string, game?: string, version?: string}} [facts]
+ * @returns {string}
+ */
+export function fillPlaceholders(markdown, facts = {}) {
+  return mapProse(markdown, (text, line) => text.replace(ANGLE, (whole, inner, offset) => {
+    const replacement = replacementFor(text, whole, inner, offset, line, facts);
+    return replacement === null ? whole : replacement;
+  }));
+}
+
+/**
+ * Every slot `fillPlaceholders` would still replace, for tests and a last check.
+ *
+ * @param {string} markdown
+ * @returns {Array<string>}
+ */
+export function findPlaceholders(markdown) {
+  const found = [];
+  mapProse(markdown, (text, line) => {
+    for (const match of text.matchAll(ANGLE)) {
+      if (replacementFor(text, match[0], match[1], match.index, line, {}) !== null) found.push(match[0]);
+    }
+    return text;
+  });
+  return found;
+}
+
+/**
+ * The same filling over the prose fields of report.json, which the dashboard
+ * shows: titles, descriptions, repro steps and quotes.
+ *
+ * @param {Object} report
+ * @param {Object} [facts]
+ * @returns {Object}
+ */
+export function fillReportPlaceholders(report, facts = {}) {
+  if (!report || !Array.isArray(report.findings)) return report;
+  const fill = value => (typeof value === 'string' ? fillPlaceholders(value, facts) : value);
+  return {
+    ...report,
+    findings: report.findings.map(finding => {
+      if (!finding || typeof finding !== 'object') return finding;
+      const out = { ...finding };
+      for (const field of ['title', 'description', 'persona_quote']) {
+        if (field in out) out[field] = fill(out[field]);
+      }
+      if (Array.isArray(out.repro_steps)) out.repro_steps = out.repro_steps.map(fill);
+      return out;
+    })
+  };
+}
+
+export default { fillPlaceholders, findPlaceholders, fillReportPlaceholders, NOT_RECORDED };

package/src/run/resume.js

@@ -3,6 +3,7 @@ import { getDeps } from './deps.js';
 import EXIT from './exit.js';
 import { runDir as runDirFor } from './paths.js';
 import { buildPartialReport, readTranscript, shouldFinalize } from './regenerate.js';
+import { reconcileDroppedFindings } from './dropped.js';
 import { createReportValidator, writeReportFiles } from './report.js';
 import * as stateFile from './state.js';
 import { playJob, resolvePersonas } from '../commands/run.js';
@@ -221,7 +222,7 @@ async function finalizeFromTranscript({ deps, api, gameId, jobId, repoRoot, row,
     // `interrupted` can go to `reporting`, which is where this run actually is:
     // it has a report now. Then `uploading`, then the delivery.
     await api.runs.transition(gameId, jobId, row.run_id, 'reporting', { reason: 'rebuilt from transcript' });
-    await deps.finalizeRun(directory, {
+    const answer = await deps.finalizeRun(directory, {
       api,
       gameId,
       jobId,
@@ -237,6 +238,20 @@ async function finalizeFromTranscript({ deps, api, gameId, jobId, repoRoot, row,
         report_md: markdown
       }
     });
+    if (answer && answer.dropped_findings > 0) {
+      log(`run ${row.run_id}: the server dropped ${answer.dropped_findings} finding(s) with no usable evidence.`);
+      try {
+        await reconcileDroppedFindings({
+          directory,
+          droppedCount: answer.dropped_findings,
+          droppedIds: answer.dropped_finding_ids,
+          options,
+          journal: { jobId, runId: row.run_id, repoRoot }
+        });
+      } catch (markError) {
+        log(`run ${row.run_id}: could not mark the dropped finding(s) in the local report (${String(markError && markError.message)}).`);
+      }
+    }
   } catch (error) {
     log(`run ${row.run_id}: could not be finalised (${String(error && error.message)}); it will be replayed.`);
     return false;

package/src/run/synthetic.js

@@ -154,6 +154,9 @@ export class SyntheticSession {
   }
 }
 
+/** The run summary's line when the page's own SDK sends the run's telemetry. */
+export const PAGE_SDK_REASON = "synthetic telemetry comes from the game's own SDK on the page, sent with the run's playtest token";
+
 /**
  * Emit the three events a run produces, best effort.
  *
@@ -161,6 +164,8 @@ export class SyntheticSession {
  * @param {string} args.apiUrl
  * @param {string|null} args.ingestKey
  * @param {string|null} args.playtestToken
+ * @param {boolean} [args.pageSdk] the driver injected the playtest token into
+ *   the page (the web driver), so the game's own SDK sends with it
  * @param {Object} args.context
  * @param {Object} args.report the run's report.json
  * @param {string} args.driver
@@ -168,8 +173,14 @@ export class SyntheticSession {
  * @param {(message: string) => void} [args.log]
  * @returns {Promise<{sent: boolean, accepted: number, reason?: string}>}
  */
-export async function emitRunEvents({ apiUrl, ingestKey, playtestToken, context, report, driver, fetchImpl, log }) {
+export async function emitRunEvents({ apiUrl, ingestKey, playtestToken, pageSdk = false, context, report, driver, fetchImpl, log }) {
   if (!ingestKey) {
+    // A web build carries its own SDK, and the web driver injected the run's
+    // playtest token into the page, so the game's own events already arrive
+    // marked synthetic. Saying "nothing was sent" there would be wrong.
+    if (pageSdk && playtestToken) {
+      return { sent: false, accepted: 0, pageSdk: true, reason: PAGE_SDK_REASON };
+    }
     return { sent: false, accepted: 0, reason: 'no ingest key configured, so no synthetic telemetry was sent' };
   }
   if (!playtestToken) {
@@ -182,6 +193,8 @@ export async function emitRunEvents({ apiUrl, ingestKey, playtestToken, context,
 
     session.track('playtest_run_started', { driver });
     for (const finding of Array.isArray(report && report.findings) ? report.findings : []) {
+      // The server kept no record of a dropped finding, so neither does the game's telemetry.
+      if (finding && finding.dropped === true) continue;
       session.track('playtest_finding', {
         finding_id: finding.id,
         severity: finding.severity,

package/src/run/tools.js

@@ -86,7 +86,7 @@ export const PLAY_TOOLS = Object.freeze([
   },
   {
     name: 'type',
-    description: 'Type text. Spends an action. Set submit to press Enter afterwards.',
+    description: 'Type text. Spends an action. Set submit to press Enter afterwards. To correct or clear a text field, set replace: the field is emptied first and holds only this text afterwards (an empty text clears it). Do that rather than a select-all key chord, which differs between platforms.',
     input_schema: {
       type: 'object',
       additionalProperties: false,
@@ -94,6 +94,8 @@ export const PLAY_TOOLS = Object.freeze([
       properties: {
         text: { type: 'string' },
         submit: { type: 'boolean' },
+        replace: { type: 'boolean', description: 'Empty the field first, so it holds only this text. playwright_web only; other drivers refuse it.' },
+        target: { type: 'string', description: 'playwright_web only, other drivers refuse it: the field to type into, named like a click target, such as textbox "Email". Without it, the focused field.' },
         thought_in_character: thought,
         note
       }
@@ -101,7 +103,7 @@ export const PLAY_TOOLS = Object.freeze([
   },
   {
     name: 'key',
-    description: 'Press one key. Spends an action. Names are the usual ones: Enter, Space, ArrowUp, Escape, a, 1.',
+    description: 'Press one key. Spends an action. Names are the usual ones: Enter, Space, ArrowUp, Escape, a, 1. Hold a modifier with +, such as Shift+Tab; editing shortcuts such as Control+A are sent with this platform\'s own modifier. To clear a text field, use type with replace instead.',
     input_schema: {
       type: 'object',
       additionalProperties: false,
@@ -187,8 +189,12 @@ export function actionFor(toolUse) {
       return Number.isFinite(Number(input.x)) && Number.isFinite(Number(input.y))
         ? { kind: 'click', target: { x: Number(input.x), y: Number(input.y) } }
         : { kind: 'click', target: String(input.target || '') };
-    case 'type':
-      return { kind: 'type', text: String(input.text || ''), submit: Boolean(input.submit) };
+    case 'type': {
+      const action = { kind: 'type', text: String(input.text || ''), submit: Boolean(input.submit) };
+      if (input.replace === true) action.replace = true;
+      if (typeof input.target === 'string' && input.target.trim()) action.target = input.target;
+      return action;
+    }
     case 'key':
       return { kind: 'key', key: String(input.key || '') };
     case 'navigate':

package/src/states.js

@@ -90,8 +90,10 @@ export const RUN_TRANSITIONS = Object.freeze({
   queued: ['launching', 'canceled', 'skipped', 'failed', 'interrupted'],
   launching: ['playing', 'failed', 'canceled', 'interrupted'],
   playing: ['reporting', 'failed', 'canceled', 'budget_exceeded', 'interrupted'],
-  reporting: ['uploading', 'failed', 'interrupted'],
-  uploading: ['succeeded', 'failed', 'interrupted'],
+  // A run out of money in its report step uploads its partial report and
+  // ends here; the API allows it from 0.2.7's companion server change.
+  reporting: ['uploading', 'failed', 'budget_exceeded', 'interrupted'],
+  uploading: ['succeeded', 'failed', 'budget_exceeded', 'interrupted'],
   interrupted: ['launching', 'playing', 'reporting', 'uploading', 'failed', 'canceled'],
   succeeded: [],
   failed: [],