ravensight-playtest 0.2.7 → 0.2.9

package/src/run/dropped.js

@@ -0,0 +1,294 @@
+import { readFile, stat } from 'node:fs/promises';
+import path from 'node:path';
+import { sha256File } from '../fsutil.js';
+import { planRunDir } from '../upload/index.js';
+import { record, STATUS } from '../upload/queue.js';
+import { TRANSCRIPT_PATH, VIDEO_PATHS } from '../upload/allowlist.js';
+import { RUN_FILES } from './paths.js';
+import { fillPlaceholders } from './placeholders.js';
+import { writeReportFiles } from './report.js';
+
+/**
+ * Findings the server dropped, marked in the local report so it agrees with
+ * the dashboard.
+ *
+ * At `complete` the server keeps a finding only when at least one of its
+ * evidence refs resolves to a file the run actually uploaded, and answers
+ * with how many it dropped, not which. Without this step the CLI said "the
+ * server dropped 1 finding" while report.md and report.json still listed it,
+ * major severity and all, and the dashboard showed one fewer.
+ *
+ * The CLI uploaded the files itself, so it can apply the server's rule to
+ * the same list and name them. When its count matches the server's, each
+ * one is marked `dropped: true` in report.json and moved in report.md to a
+ * section of its own: kept for the developer to read, out of the top issues,
+ * the aggregate and the synthetic telemetry. When the counts disagree it
+ * does not guess which; it says how many the server dropped and points at
+ * the dashboard.
+ */
+
+/** The section dropped findings are moved to. */
+export const DROPPED_HEADING = '## Dropped by the server (no usable evidence)';
+
+/**
+ * The uploaded path, or candidate paths, an evidence entry needs. The same
+ * mapping as the server's `manifestPathForEvidence`.
+ *
+ * @param {{type?: string, ref?: string}} evidence
+ * @returns {string|Array<string>|null}
+ */
+export function evidencePath(evidence) {
+  if (!evidence || typeof evidence.type !== 'string') return null;
+  const ref = typeof evidence.ref === 'string' ? evidence.ref : '';
+  switch (evidence.type) {
+    case 'screenshot':
+      return ref || null;
+    case 'video_ts':
+      return [...VIDEO_PATHS];
+    case 'transcript_step':
+      return TRANSCRIPT_PATH;
+    case 'file_line':
+      return ref ? ref.split(':')[0] : null;
+    default:
+      return null;
+  }
+}
+
+/**
+ * The findings none of whose evidence resolves to an uploaded file.
+ *
+ * @param {Object} report
+ * @param {Iterable<string>} uploaded the run relative paths that were uploaded
+ * @returns {Array<Object>}
+ */
+export function unbackedFindings(report, uploaded) {
+  const paths = new Set(uploaded);
+  const findings = Array.isArray(report && report.findings) ? report.findings : [];
+  return findings.filter(finding => {
+    if (!finding || finding.dropped === true) return false;
+    const evidence = Array.isArray(finding.evidence) ? finding.evidence : [];
+    return !evidence.some(entry => {
+      const wanted = evidencePath(entry);
+      if (wanted === null) return false;
+      return (Array.isArray(wanted) ? wanted : [wanted]).some(candidate => paths.has(candidate));
+    });
+  });
+}
+
+/**
+ * report.json with the named findings marked.
+ *
+ * @param {Object} report
+ * @param {Set<string>} ids
+ * @returns {Object}
+ */
+export function markDroppedFindings(report, ids) {
+  return {
+    ...report,
+    findings: (report.findings || []).map(finding => (finding && ids.has(finding.id) ? { ...finding, dropped: true } : finding))
+  };
+}
+
+const SECTION = /^## /;
+const LIST_ITEM = /^\s*(?:[-*+]|\d+[.)])\s+/;
+
+/**
+ * report.md with the dropped findings out of the summary and in a section of
+ * their own.
+ *
+ * A list item that names a dropped finding id and no kept one is removed
+ * wherever it is, which is how "Top issues" names them (`F003 ...`). A
+ * section left with nothing under it says so. Then the dropped findings are
+ * listed under {@link DROPPED_HEADING}, replacing one written before, so a
+ * second reconcile does not stack them.
+ *
+ * @param {string} markdown
+ * @param {Array<Object>} dropped the dropped findings, from report.json
+ * @param {Array<Object>} kept the findings the server kept
+ * @param {{unidentified?: number}} [opts] a count the CLI could not attribute
+ * @returns {string}
+ */
+export function rewriteMarkdown(markdown, dropped, kept, { unidentified = 0 } = {}) {
+  const lines = withoutDroppedSection(String(markdown || '').replace(/\s+$/, '')).split('\n');
+  const droppedIds = dropped.map(finding => String(finding.id)).filter(Boolean);
+  const keptIds = kept.map(finding => String(finding.id)).filter(Boolean);
+  const names = (line, id) => new RegExp(`(^|[^A-Za-z0-9])${escapeRegExp(id)}(?![0-9])`).test(line);
+
+  const out = [];
+  let section = null;
+  let sectionRemoved = 0;
+  let sectionKept = 0;
+  const closeSection = () => {
+    if (section !== null && sectionRemoved > 0 && sectionKept === 0) {
+      // Trailing blank lines stay after the note, as they were after the list.
+      let blanks = 0;
+      while (out.length > 0 && out[out.length - 1].trim() === '') {
+        out.pop();
+        blanks += 1;
+      }
+      out.push('- Nothing here that the server kept. See the dropped findings below.');
+      for (let i = 0; i < blanks; i += 1) out.push('');
+    }
+  };
+  for (const line of lines) {
+    if (SECTION.test(line)) {
+      closeSection();
+      section = line;
+      sectionRemoved = 0;
+      sectionKept = 0;
+      out.push(line);
+      continue;
+    }
+    if (LIST_ITEM.test(line) && droppedIds.some(id => names(line, id)) && !keptIds.some(id => names(line, id))) {
+      sectionRemoved += 1;
+      continue;
+    }
+    if (LIST_ITEM.test(line)) sectionKept += 1;
+    out.push(line);
+  }
+  closeSection();
+
+  const tail = ['', DROPPED_HEADING];
+  if (dropped.length > 0) {
+    tail.push(
+      'The server keeps a finding only when at least one of its evidence references points at a file this run uploaded. These did not, so the dashboard does not show them and they are not counted. They are kept here so you can judge them yourself.',
+      ''
+    );
+    for (const finding of dropped) {
+      const evidence = (Array.isArray(finding.evidence) ? finding.evidence : [])
+        .map(entry => `${entry && entry.type} ${entry && entry.ref}`)
+        .join(', ') || 'none';
+      // The title is the model's prose, copied here; fill any slot it carries first.
+      tail.push(`- ${finding.id} (${finding.severity}, ${finding.category}): ${fillPlaceholders(String(finding.title || ''))}. Evidence cited: ${evidence}.`);
+    }
+  }
+  if (unidentified > 0) {
+    tail.push(
+      `The server dropped ${unidentified} finding(s) with no usable evidence, and this report could not tell which, so every finding above is still listed and counted here. The dashboard shows the findings the server kept.`
+    );
+  }
+  return `${out.join('\n')}\n${tail.join('\n')}\n`;
+}
+
+/**
+ * Remove a dropped section written by an earlier reconcile.
+ *
+ * @param {string} markdown
+ * @returns {string}
+ */
+function withoutDroppedSection(markdown) {
+  const lines = markdown.split('\n');
+  const start = lines.findIndex(line => line.trim() === DROPPED_HEADING);
+  if (start === -1) return markdown;
+  let end = lines.findIndex((line, index) => index > start && SECTION.test(line));
+  if (end === -1) end = lines.length;
+  return [...lines.slice(0, start), ...lines.slice(end)].join('\n').replace(/\s+$/, '');
+}
+
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/**
+ * Mark the findings the server dropped in a run directory's report files.
+ *
+ * The server's own list (`dropped_finding_ids`) is used when it sends one.
+ * Without it the CLI applies the server's evidence rule to its upload plan,
+ * and marks only when that count matches the server's. Findings marked by an
+ * earlier reconcile count towards the server's number, so running it twice
+ * changes nothing.
+ *
+ * Rewriting the two files changes their sha256, which would make `upload
+ * <job>` send them again over the validated artifacts. So each new hash is
+ * recorded in the job's upload journal as a local edit, which the journal
+ * treats as already delivered.
+ *
+ * @param {Object} args
+ * @param {string} args.directory the run directory
+ * @param {number} args.droppedCount `dropped_findings` from the complete answer
+ * @param {Array<string>} [args.droppedIds] `dropped_finding_ids` from the complete answer, when the server sends it
+ * @param {{uploadVideo?: boolean, uploadTranscript?: boolean}} [args.options] what was uploaded
+ * @param {{jobId: string, runId: string, repoRoot?: string}} [args.journal] where to record the local edit
+ * @param {(dir: string, opts: Object) => Promise<{files: Array<{path: string}>}>} [args.plan] test seam
+ * @returns {Promise<{report: Object, markdown: string, identified: boolean, dropped: Array<string>, source: string}|null>}
+ *   null when there was nothing to mark or no report on disk
+ */
+export async function reconcileDroppedFindings({ directory, droppedCount, droppedIds = null, options = {}, journal = null, plan = planRunDir }) {
+  const serverIds = Array.isArray(droppedIds) ? droppedIds.map(String).filter(Boolean) : null;
+  const count = serverIds && serverIds.length > 0 ? serverIds.length : Number(droppedCount) || 0;
+  if (count <= 0) return null;
+  let report;
+  let markdown;
+  try {
+    report = JSON.parse(await readFile(path.join(directory, RUN_FILES.reportJson), 'utf8'));
+    markdown = await readFile(path.join(directory, RUN_FILES.reportMd), 'utf8');
+  } catch {
+    return null;
+  }
+  if (!report || !Array.isArray(report.findings)) return null;
+
+  const already = report.findings.filter(finding => finding && finding.dropped === true).map(finding => finding.id);
+  let ids;
+  let identified;
+  let source;
+  if (serverIds && serverIds.length > 0) {
+    const present = new Set(report.findings.filter(Boolean).map(finding => finding.id));
+    ids = new Set(serverIds.filter(id => present.has(id)));
+    identified = ids.size === serverIds.length;
+    source = 'server';
+  } else {
+    const planned = await plan(directory, { video: Boolean(options.uploadVideo), transcript: Boolean(options.uploadTranscript) });
+    const unbacked = unbackedFindings(report, planned.files.map(file => file.path)).map(finding => finding.id);
+    identified = already.length + unbacked.length === count;
+    ids = new Set(identified ? [...already, ...unbacked] : already);
+    source = 'plan';
+  }
+  const marked = markDroppedFindings(report, ids);
+  const dropped = marked.findings.filter(finding => finding && finding.dropped === true);
+  const kept = marked.findings.filter(finding => !(finding && finding.dropped === true));
+  const rewritten = rewriteMarkdown(markdown, dropped, kept, { unidentified: identified ? 0 : count - dropped.length });
+  await writeReportFiles({ runDirectory: directory, markdown: rewritten, report: marked });
+  await recordLocalEdits(directory, journal);
+  return { report: marked, markdown: rewritten, identified, dropped: [...ids], source };
+}
+
+/**
+ * Record the rewritten report files' hashes as local edits, so the upload
+ * journal treats them as delivered rather than owed.
+ *
+ * @param {string} directory
+ * @param {{jobId: string, runId: string, repoRoot?: string}|null} journal
+ */
+async function recordLocalEdits(directory, journal) {
+  const where = journal && journal.jobId && journal.repoRoot ? journal : journalFromDirectory(directory) || (journal && journal.jobId ? journal : null);
+  if (!where) return;
+  for (const name of [RUN_FILES.reportMd, RUN_FILES.reportJson]) {
+    const file = path.join(directory, name);
+    const info = await stat(file);
+    await record(where.jobId, {
+      kind: 'run',
+      run_id: where.runId,
+      path: name,
+      sha256: await sha256File(file),
+      size: info.size,
+      status: STATUS.LOCAL_EDIT,
+      detail: 'dropped findings marked after complete; the delivered copy stays the record'
+    }, where.repoRoot ? { repoRoot: where.repoRoot } : {});
+  }
+}
+
+/**
+ * The job, run and repository a run directory belongs to, from its own path
+ * (`<repo>/.ravensight/jobs/<jobId>/runs/<runId>`), or null for any other.
+ *
+ * @param {string} directory
+ * @returns {{jobId: string, runId: string, repoRoot: string}|null}
+ */
+export function journalFromDirectory(directory) {
+  const parts = path.resolve(directory).split(path.sep);
+  const n = parts.length;
+  if (n < 6 || parts[n - 2] !== 'runs' || parts[n - 4] !== 'jobs' || parts[n - 5] !== '.ravensight') return null;
+  return { jobId: parts[n - 3], runId: parts[n - 1], repoRoot: parts.slice(0, n - 5).join(path.sep) || path.sep };
+}
+
+export default { reconcileDroppedFindings, unbackedFindings, markDroppedFindings, rewriteMarkdown, evidencePath, DROPPED_HEADING };

package/src/run/heartbeat.js

@@ -58,6 +58,11 @@ export function createHeartbeat({
   let canceled = false;
   let failures = 0;
   let inFlight = false;
+  // Set by stop(). A beat already on the wire when the run ends must not
+  // write its progress after the run's final state: in 0.2.8 one landed
+  // after the outcome and put a finished run back to `playing` in the
+  // journal.
+  let stopped = false;
   // The progress the loop last reported. The interval beat carries it too,
   // rather than resetting the server's and the journal's idea of the
   // checkpoint to zero every five seconds.
@@ -78,7 +83,7 @@ export function createHeartbeat({
     try {
       const answer = await api.runs.heartbeat(gameId, jobId, runId, { heartbeat: true, ...progress });
       failures = 0;
-      if (onBeat) await onBeat({ ...progress });
+      if (onBeat && !stopped) await onBeat({ ...progress });
       if (answer && answer.run && answer.run.usage && onUsage) onUsage(answer.run.usage);
       if (answer && answer.cancel_requested && !canceled) {
         canceled = true;
@@ -111,6 +116,7 @@ export function createHeartbeat({
     },
 
     start() {
+      stopped = false;
       if (timer) return this;
       timer = setIntervalImpl(() => {
         beat().catch(() => {});
@@ -121,6 +127,7 @@ export function createHeartbeat({
     },
 
     stop() {
+      stopped = true;
       if (timer) clearIntervalImpl(timer);
       timer = null;
       return this;

package/src/run/personaLoop.js

@@ -2,15 +2,15 @@ import { selectAction, JEV_CONTROLLER_PROMPT } from './actionSelector.js';
 import { mkdir, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import { prepareMessages, MAX_IMAGES_IN_CONTEXT, KEEP_OBSERVATIONS, shrinkImages } from './context.js';
-import { observationHash } from './drivers/driver.js';
 import { withRollingCacheBreakpoint } from './fixLoop.js';
 import { callStepWithRetry, ProxyError, STEPS } from './model.js';
 import { RUN_FILES } from './paths.js';
-import { checkEvidence, createReportValidator, lintMarkdownSections, repairInstruction, REQUIRED_MARKDOWN_SECTIONS, stampReport, writeReportFiles } from './report.js';
+import { checkEvidence, createReportValidator, lintMarkdownSections, normaliseEvidence, repairInstruction, REQUIRED_MARKDOWN_SECTIONS, stampReport, writeReportFiles } from './report.js';
 import { assertNoSecrets } from './secretScan.js';
 import { ACTION_TOOLS, PLAY_TOOLS, SWIPE_TOOL, REPORT_TOOLS, executeTool } from './tools.js';
 import { createTranscript } from './transcript.js';
 import { createUsageLedger, writeUsageFile } from './usage.js';
+import { fillPlaceholders, fillReportPlaceholders } from './placeholders.js';
 
 /**
  * One persona's playthrough: PLAYING then REPORTING (spec 03).
@@ -387,7 +387,7 @@ async function playLoop({
   latestObservation = first.text;
   visualObservation = Boolean(first.screenshotBase64);
   transcript.write({ kind: 'observation', step: first.step, text: first.text, url: first.url });
-  const landing = await safeScreenshot(driver, 'landing', screenshots, transcript);
+  const landing = await safeScreenshot(driver, 'landing', screenshots, transcript, observations.length);
   messages.push({
     role: 'user',
     content: [
@@ -502,7 +502,7 @@ async function playLoop({
           transcript.write({ kind: 'note', tool: toolUse.name, note });
         },
         onScreenshot: shot => {
-          screenshots.push({ ...shot, order: nextOrder() });
+          screenshots.push({ ...shot, observation: observations.length, order: nextOrder() });
           transcript.write({ kind: 'screenshot', path: shot.relativePath, reason: shot.reason });
         }
       });
@@ -551,7 +551,7 @@ async function playLoop({
       // with the persona's view of the screen in front of it. A 4xx on a
       // resource is a `network` note, which the report step reads as context
       // rather than as a bug (a missing favicon is not a broken game).
-      const shot = await safeScreenshot(driver, 'console-error', screenshots, transcript);
+      const shot = await safeScreenshot(driver, 'console-error', screenshots, transcript, observations.length);
       for (const entry of consoleErrors) {
         const kind = RESOURCE_4XX.test(String(entry.text)) ? 'network' : 'bug';
         const note = { kind, text: `${entry.type}: ${entry.text}`, automatic: true, ...(shot ? { screenshot: shot.relativePath } : {}) };
@@ -591,7 +591,7 @@ async function playLoop({
     quitDetail = `the action budget of ${maxActions} ran out`;
   }
 
-  const final = await safeScreenshot(driver, 'final', screenshots, transcript);
+  const final = await safeScreenshot(driver, 'final', screenshots, transcript, observations.length);
   return {
     actionsTaken: actions,
     turns,
@@ -650,11 +650,15 @@ async function reportPhase({
         observationLines(observations),
         '',
         'Each note carries "after": the URL and title of the observation that followed it. A blocker or major finding must cite the observation index where the failure is visible, as evidence {"type": "transcript_step", "ref": "observation N"}, and must not be contradicted by its note\'s "after" state or by a later persona note: if the next screen or a later note shows the game working, it is not a blocker or a major. Cite a note as {"type": "transcript_step", "ref": "note N"}.',
+        'A blocker or major finding must also cite the screenshot taken at or after that observation (the list below says which observation each one followed), and that screenshot must not contradict it: if it shows the thing the finding says is missing or broken, such as the field or the button, it is not a blocker or a major. A text observation can be cut short on a long page; the screenshot is what the player saw.',
         '',
-        'Never write calendar dates or times in the prose of report.md or report.json. The runner stamps when the report was generated.',
+        'Every evidence entry is an object with exactly two keys, "type" and "ref", and nothing else. A screenshot is {"type": "screenshot", "ref": "screenshots/04-example.png"}, never "path" or "file".',
+        '',
+        'Never write calendar dates or times in the prose of report.md or report.json. The runner stamps when the report was generated, so leave the template\'s Date out rather than writing a placeholder for it.',
+        'Replace every <...> slot in the template with real content. Never copy a <...> placeholder into the report.',
         '',
         'Screenshots available as evidence (use these exact paths in evidence refs):',
-        screenshots.length > 0 ? screenshots.map(shot => `- ${shot.relativePath}${shot.reason ? ` (${shot.reason})` : ''}`).join('\n') : '(none)',
+        screenshots.length > 0 ? screenshots.map(screenshotLine).join('\n') : '(none)',
         '',
         `report.md must contain these section headings, spelled exactly: ${REQUIRED_MARKDOWN_SECTIONS.map(section => `"${section}"`).join(', ')}.`,
         'Write report.md first, then report.json. Leave usage as zeros; the runner fills it in.'
@@ -665,12 +669,20 @@ async function reportPhase({
   let report = null;
   let markdown = null;
   let lint = [];
-
-  // Two attempts at most: the first, and one repair if the schema or the
-  // evidence refs refuse it. Each attempt is a metered question, and the
-  // dogfood gate asks for "first or the repair attempt", not an
-  // open ended loop.
-  for (let attempt = 1; attempt <= 2; attempt += 1) {
+  // The newest report.json the model wrote, across turns: a repair turn that
+  // rewrites only report.md keeps the report.json it already wrote.
+  let jsonText = null;
+  let repaired = false;
+  let continued = false;
+
+  // One repair at most when the schema or the evidence refs refuse what was
+  // written: each turn is a metered question, and the dogfood gate asks for
+  // "first or the repair attempt", not an open ended loop. A turn that wrote
+  // report.md and stopped before report.json is not a refused report, so it
+  // gets one plain "now write report.json" that does not spend the repair.
+  // In 0.2.8 it did, and the report.json that came back had its five
+  // screenshot refs under "path" with no turn left to fix them.
+  for (let attempt = 1; ; attempt += 1) {
     const answer = await callStepWithRetry({
       client,
       jobId,
@@ -693,12 +705,13 @@ async function reportPhase({
       files.set(name, String(toolUse.input.content || ''));
     }
     if (files.has(RUN_FILES.reportMd)) markdown = files.get(RUN_FILES.reportMd);
+    if (files.has(RUN_FILES.reportJson)) jsonText = files.get(RUN_FILES.reportJson);
 
     let parsed = null;
     let parseError = null;
-    if (files.has(RUN_FILES.reportJson)) {
+    if (jsonText !== null) {
       try {
-        parsed = JSON.parse(files.get(RUN_FILES.reportJson));
+        parsed = JSON.parse(jsonText);
       } catch (error) {
         parseError = String(error && error.message);
       }
@@ -718,7 +731,13 @@ async function reportPhase({
       quitReason: play.quitReason,
       quitDetail: play.quitDetail
     };
-    const candidate = parsed ? capNetworkOnlyFindings(stampReport(parsed, facts), { notes, screenshots }) : null;
+    // An evidence path under the wrong key is moved to `ref` before anything
+    // judges the report; see normaliseEvidence.
+    const normalised = parsed ? normaliseEvidence(parsed, runDirectory) : null;
+    if (normalised && normalised.changed > 0) {
+      transcript.write({ kind: 'report_evidence_normalised', attempt, entries: normalised.changed });
+    }
+    const candidate = normalised ? capNetworkOnlyFindings(stampReport(normalised.report, facts), { notes, screenshots }) : null;
     const validation = candidate ? validator.validate(candidate) : { ok: false, details: [{ instancePath: '/', message: parseError || 'report.json was not written' }] };
     const evidenceProblems = candidate ? await checkEvidence(candidate, runDirectory) : [];
     // The server lints the same four sections at `complete` and records the
@@ -740,31 +759,50 @@ async function reportPhase({
       ...evidenceProblems,
       ...missingSections.map(section => `report.md has no "${section}" section`)
     ];
-    if (attempt === 2) {
+
+    // Every tool_use the next request echoes back needs its tool_result, or
+    // the API refuses the request. A write cut off at the token limit is
+    // malformed: it was not saved, and it is answered so.
+    const toolResults = answer.toolUses.map(toolUse => (toolUse.malformed
+      ? { type: 'tool_result', tool_use_id: toolUse.id, is_error: true, content: [{ type: 'text', text: CUT_OFF_WRITE }] }
+      : { type: 'tool_result', tool_use_id: toolUse.id, content: [{ type: 'text', text: 'saved' }] }));
+
+    if (jsonText === null && files.has(RUN_FILES.reportMd) && !continued) {
+      continued = true;
+      messages.push({ role: 'assistant', content: assistantContent(answer) });
+      messages.push({ role: 'user', content: toolResults.concat([{ type: 'text', text: CONTINUE_WITH_JSON }]) });
+      continue;
+    }
+
+    if (repaired) {
       if (!candidate) throw new Error(`the report step did not produce a valid report.json: ${lint.join('; ')}`);
-      // Second attempt still imperfect: keep it, record the lint, and let
+      // Still imperfect after the repair: keep it, record the lint, and let
       // the server have the final word. It validates the same schema and
       // drops unevidenced findings itself, so this is a report with known
-      // problems rather than no report at all.
+      // problems rather than no report at all. Every problem on its own line,
+      // before anything is uploaded, so a refusal at complete is never the
+      // first the developer hears of it.
       report = candidate;
-      log(`report.json still has problems after one repair: ${lint.join('; ')}`);
+      log(`report.json still has problems after one repair (${lint.length}); the server will refuse a report that fails its schema:\n${lint.map(line => `  - ${line}`).join('\n')}`);
       break;
     }
 
+    repaired = true;
     messages.push({ role: 'assistant', content: assistantContent(answer) });
     messages.push({
       role: 'user',
-      content: answer.toolUses
-        .filter(toolUse => !toolUse.malformed)
-        .map(toolUse => ({ type: 'tool_result', tool_use_id: toolUse.id, content: [{ type: 'text', text: 'saved' }] }))
-        .concat([{ type: 'text', text: repairInstruction(validation.details, evidenceProblems, missingSections) }])
+      content: toolResults.concat([{ type: 'text', text: repairInstruction(validation.details, evidenceProblems, missingSections) }])
     });
   }
 
   if (!markdown) {
     markdown = renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason: 'The report step wrote report.json but no report.md.' });
   }
-  markdown = stampGeneratedAt(markdown, new Date().toISOString());
+  const generatedAt = new Date().toISOString();
+  const facts = reportFacts({ persona, driverName, target, play });
+  markdown = stampGeneratedAt(markdown, generatedAt, facts);
+  // The same slots in the prose the dashboard shows from report.json.
+  report = fillReportPlaceholders(report, { generatedAt, ...facts });
 
   await writeReportFiles({ runDirectory, markdown, report });
   return { report, markdown, lint };
@@ -895,10 +933,10 @@ function redactToolInput(toolUse) {
   return rest;
 }
 
-async function safeScreenshot(driver, name, screenshots, transcript) {
+async function safeScreenshot(driver, name, screenshots, transcript, observation = null) {
   try {
     const shot = await driver.screenshot(name);
-    screenshots.push({ ...shot, reason: name, order: nextOrder() });
+    screenshots.push({ ...shot, reason: name, ...(Number.isInteger(observation) ? { observation } : {}), order: nextOrder() });
     transcript.write({ kind: 'screenshot', path: shot.relativePath, reason: name });
     return shot;
   } catch (error) {
@@ -1083,14 +1121,14 @@ async function writePartialReport({ outcome, runDirectory, persona, driverName,
     play,
     reason: 'The run reached its model budget before the report step. What the persona saw is below, and report.json carries the notes as unreviewed findings.'
   });
-  const stamped = stampGeneratedAt(markdown, new Date().toISOString());
+  const stamped = stampGeneratedAt(markdown, new Date().toISOString(), reportFacts({ persona, driverName, target, play }));
   await writeReportFiles({ runDirectory, markdown: stamped, report });
   outcome.report = report;
   outcome.reportMarkdown = stamped;
 }
 
 async function writePartialMarkdown({ runDirectory, persona, driverName, target, notes, screenshots, play, reason }) {
-  const markdown = stampGeneratedAt(renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason }), new Date().toISOString());
+  const markdown = stampGeneratedAt(renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason }), new Date().toISOString(), reportFacts({ persona, driverName, target, play }));
   assertNoSecrets('report.md', markdown);
   await writeFile(path.join(runDirectory, RUN_FILES.reportMd), markdown, 'utf8');
 }
@@ -1116,6 +1154,28 @@ export function noteLine(note, i) {
   return `${i + 1}. [${note.kind}${note.severity ? `/${note.severity}` : ''}] ${note.text}${tags.length > 0 ? ` (${tags.join(', ')})` : ''}${after}`;
 }
 
+/** The tool_result for a write_file call that was cut off before its input was complete. */
+export const CUT_OFF_WRITE = 'That write_file call was cut off before it finished, so nothing was saved. Write that file again, shorter if it has to be.';
+
+/** The one follow-up a report turn gets for writing report.md and stopping. */
+export const CONTINUE_WITH_JSON = 'report.md is saved. Now write report.json with write_file. Every evidence entry is exactly {"type": ..., "ref": ...}; a screenshot is {"type": "screenshot", "ref": "screenshots/04-example.png"}.';
+
+/**
+ * A screenshot as one line of the report prompt: its path, the observation
+ * it followed (what a blocker or major has to be checked against) and why
+ * it was taken.
+ *
+ * @param {Object} shot
+ * @returns {string}
+ */
+export function screenshotLine(shot) {
+  const tags = [
+    Number.isInteger(shot.observation) && shot.observation > 0 ? `after observation ${shot.observation}` : null,
+    shot.reason ? String(shot.reason) : null
+  ].filter(Boolean);
+  return `- ${shot.relativePath}${tags.length > 0 ? ` (${tags.join('; ')})` : ''}`;
+}
+
 /** The most observations listed in the report prompt; the newest are kept. */
 export const MAX_OBSERVATIONS_IN_REPORT = 200;
 
@@ -1216,13 +1276,32 @@ export function capNetworkOnlyFindings(report, { notes = [], screenshots = [] }
  * @param {string} generatedAt an ISO timestamp
  * @returns {string}
  */
-export function stampGeneratedAt(markdown, generatedAt) {
+export function stampGeneratedAt(markdown, generatedAt, facts = {}) {
   const line = `**Generated at:** ${generatedAt}`;
   const lines = String(markdown || '').split('\n').filter(entry => !/^\*\*Generated at:\*\* /.test(entry));
   const heading = lines.findIndex(entry => /^# /.test(entry));
-  if (heading === -1) return [line, '', ...lines].join('\n');
-  lines.splice(heading + 1, 0, line);
-  return lines.join('\n');
+  if (heading === -1) lines.unshift(line, '');
+  else lines.splice(heading + 1, 0, line);
+  // The template's own slots (`**Date:** <date>`) are filled from what the
+  // runner knows, because the prompt forbids the model a calendar date and a
+  // model that obeys both writes the placeholder itself.
+  return fillPlaceholders(lines.join('\n'), { generatedAt, ...facts });
+}
+
+/**
+ * What the runner knows about a run, for the template slots a report left
+ * unfilled.
+ *
+ * @returns {{driver: string, target: string, persona: string, actionsTaken: number, quitReason: string}}
+ */
+function reportFacts({ persona, driverName, target, play }) {
+  return {
+    driver: driverName,
+    target,
+    persona: persona ? persona.name || persona.slug : undefined,
+    actionsTaken: play ? play.actionsTaken : undefined,
+    quitReason: play ? play.quitReason : undefined
+  };
 }
 
 /**