ravensight-playtest 0.2.8 → 0.2.9

package/README.md

@@ -254,7 +254,15 @@ How the report step reads what happened:
 - Every note carries `after`, the URL and title of the screen that followed
   it. A blocker or major finding has to cite the observation where the failure
   is visible and must not be contradicted by that next screen or by a later
-  persona note.
+  persona note. It also has to cite the screenshot taken at or after that
+  observation (the prompt says which observation each screenshot followed),
+  and that screenshot must not show what the finding says is missing.
+- Every evidence entry is `{"type": ..., "ref": ...}`. A screenshot path the
+  model put under `path` or `file` is moved to `ref` before the report is
+  checked; the file still has to exist in the run directory. If the server
+  still refuses `report.json` at `complete`, the CLI prints every validation
+  message it sent, the run is not counted as succeeded or in the aggregate,
+  and the job exits 1.
 - The model is told never to write calendar dates. `report.md` carries the
   CLI's own `Generated at` timestamp under its title, and the aggregate report's
   header uses its `generated_at`.
@@ -285,6 +293,12 @@ macOS, Windows and Linux; ordinary typing is keystrokes. An editing shortcut
 such as select all uses the platform's own modifier (Command on a Mac, Control
 elsewhere, or `--key-modifier`).
 
+On a long web page the persona's text view of the screen is capped at 4,000
+characters. Over the cap, buttons, links and fields are kept first, a long run
+of look-alike items (a list of 249 cards) is shown as its first two and "and
+247 more like it", and what changed since the last look comes next, so a form
+after a long list still reaches the persona.
+
 Each persona starts at the game's URL as a fresh player with no account, so a
 sign-up form with validation can use up a whole unit before the game itself is
 reached. Point `--build-url` at a guest or demo path, or say where to start in
@@ -440,6 +454,10 @@ lost to it. The CLI lints locally first and feeds any miss back into one repair
 turn, which costs a model turn; a report written with the four headings from the
 start avoids that.
 
+The repository's tests check rendered reports against `test/fixtures/server/`, a
+byte copy of the server's report schema from game_tracker commit 864027e,
+which has to be refreshed by hand whenever the server's schema changes.
+
 ## Exit codes
 
 | Code | Meaning |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ravensight-playtest",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "description": "AI personas play your game and file reports to Ravensight.",
   "license": "MIT",
   "homepage": "https://ravensight.io/docs/#playtest",

package/src/commands/run.js

@@ -445,7 +445,10 @@ export async function playJob({
   const reportBudgetRefused = results.some(result => result.reportStepBudget === 'refused');
   const reportBudgetDelivered = results.some(result => result.reportStepBudget === 'delivered' || result.reportStepBudget === 'dry_upload');
   const canceled = results.some(result => result.state === 'canceled');
-  const anySucceeded = results.some(result => result.state === 'succeeded');
+  // A run whose report the server refused played, but the server never
+  // completed it: it is not a success, and CI should hear about it.
+  const anySucceeded = results.some(result => result.state === 'succeeded' && !result.reportRefused);
+  const reportRefused = results.some(result => result.reportRefused);
 
   if (finish) {
     const finished = await api.jobs.finish(gameId, jobId, {
@@ -459,7 +462,7 @@ export async function playJob({
   log(`Reports: ${directory}`);
 
   if (canceled) return EXIT.CANCELED;
-  if (reportBudgetRefused) return EXIT.FAILED;
+  if (reportBudgetRefused || reportRefused) return EXIT.FAILED;
   if (playBudgetHit) return EXIT.BUDGET;
   return anySucceeded || reportBudgetDelivered ? EXIT.OK : EXIT.FAILED;
 }
@@ -620,7 +623,15 @@ export async function playPersonaRun({
     budget_warned: outcome.budgetWarned
   });
 
-  await deliver({ deps, api, gameId, jobId, runId, directory, outcome, options, log, journal, repoRoot });
+  const delivered = {};
+  await deliver({ deps, api, gameId, jobId, runId, directory, outcome, options, log, journal, repoRoot, delivery: delivered });
+
+  if (delivered.reportInvalid) {
+    // No synthetic telemetry for findings the server does not hold, and no
+    // "succeeded" line for a run the server never completed.
+    log(`run ${runId} (${persona.slug}) played ${outcome.actionsTaken} actions (${outcome.quitReason}), but the server refused its report, so the run is not complete.`);
+    return { runId, persona: persona.slug, state: outcome.state, quitReason: outcome.quitReason, reportRefused: true };
+  }
 
   if (outcome.report) {
     const emitted = await emitRunEvents({
@@ -713,7 +724,7 @@ export async function deliver({ deps, api, gameId, jobId, runId, directory, outc
       const { report: ignored, report_md: alsoIgnored, ...withoutReport } = body;
       answer = await api.runs.complete(gameId, jobId, runId, withoutReport);
     }
-    if (journal) await stateFile.patchRun(repoRoot, journal, runId, { uploaded: true, completed: true });
+    if (journal) await stateFile.patchRun(repoRoot, journal, runId, { uploaded: true, completed: true, report_refused: false });
     if (answer && answer.dropped_findings > 0) {
       log(`run ${runId}: the server dropped ${answer.dropped_findings} finding(s) with no usable evidence.`);
       await markDropped({ directory, jobId, runId, repoRoot, outcome, droppedCount: answer.dropped_findings, droppedIds: answer.dropped_finding_ids, options, log });
@@ -728,11 +739,36 @@ export async function deliver({ deps, api, gameId, jobId, runId, directory, outc
       delivery.status = error && error.status;
       delivery.code = error && error.code;
     }
+    if (error && error.status === 422 && error.code === 'invalid_report') {
+      // Every validation message first, one per line, before anything says
+      // the run did not complete: the summary line alone ("does not match
+      // the playtest report schema") is not something anyone can act on.
+      const details = refusalDetails(error);
+      log(`run ${runId}: the server refused report.json (${details.length} problem(s)):\n${details.length > 0 ? details.map(line => `  - ${line}`).join('\n') : '  - (the server named no problem)'}`);
+      if (delivery) delivery.reportInvalid = true;
+      if (journal) await stateFile.patchRun(repoRoot, journal, runId, { report_refused: true });
+      log(`run ${runId}: not completed, so the server keeps no findings from it; whether its unit is refunded is decided when the job settles. report.json and the screenshots stay in ${directory}.`);
+      return null;
+    }
     log(`run ${runId}: upload or complete was refused (${String(error && error.message)}). The artifacts are in the run directory and "ravensight-playtest resume" can retry it.`);
     return null;
   }
 }
 
+/**
+ * The validation messages a 422 carried, as `pointer: message` lines. Read
+ * from `details` (the API client's error) or `body.details` (the envelope).
+ *
+ * @param {any} error
+ * @returns {Array<string>}
+ */
+export function refusalDetails(error) {
+  const list = Array.isArray(error && error.details) && error.details.length > 0
+    ? error.details
+    : (error && error.body && Array.isArray(error.body.details) ? error.body.details : []);
+  return list.map(detail => `${(detail && detail.instancePath) || '/'}: ${(detail && detail.message) || 'is invalid'}`);
+}
+
 /**
  * Bring the local report into line with what the server kept, so report.md,
  * report.json, the aggregate and the telemetry count what the dashboard
@@ -843,7 +879,7 @@ async function aggregatePhase({
 
   // The aggregate reads reports off disk, so a resumed job whose earlier runs
   // were played by a previous process is covered too.
-  const reports = await loadRunReports(repoRoot, jobId, personaRuns);
+  const reports = await loadRunReports(repoRoot, jobId, personaRuns, journal);
   let aggregate;
   try {
     aggregate = await buildAggregate({
@@ -1107,9 +1143,16 @@ export function resolveKeyModifier(value, env = process.env) {
  * from the notes when the money ran out) is loaded as a `budget_exceeded`
  * run: its findings count, and the aggregate says the job is partial.
  */
-export async function loadRunReports(repoRoot, jobId, personaRuns) {
+export async function loadRunReports(repoRoot, jobId, personaRuns, journal = null) {
   const out = [];
   for (const entry of personaRuns) {
+    // A report the server refused is on disk but not in the record: the
+    // summary must not count findings the dashboard does not have.
+    const recorded = journal && journal.runs ? journal.runs[entry.run_id] : null;
+    if (recorded && recorded.report_refused === true) {
+      out.push({ runId: entry.run_id, persona: entry.persona, state: 'failed', report: null, reportRefused: true });
+      continue;
+    }
     const file = path.join(runDirFor(repoRoot, jobId, entry.run_id), RUN_FILES.reportJson);
     try {
       const report = JSON.parse(await readFile(file, 'utf8'));

package/src/run/drivers/web.js

@@ -77,9 +77,22 @@ const INTERACTIVE_ROLES = new Set([
  * @returns {string}
  */
 export function compactAriaSnapshot(text) {
+  const nodes = compactAriaNodes(text);
+  return nodes === null ? String(text == null ? '' : text) : nodes.map(node => node.line).join('\n');
+}
+
+/**
+ * The kept nodes of an aria snapshot with the depth each one had, or null for
+ * text that is not an aria snapshot. The depth is not shown to the model; it
+ * is what tells repeated siblings apart from a panel that follows them.
+ *
+ * @param {string} text
+ * @returns {Array<{depth: number, line: string, body: string, role: string}>|null}
+ */
+export function compactAriaNodes(text) {
   const value = String(text == null ? '' : text);
   const lines = value.split('\n');
-  if (!lines.some(line => /^\s*- /.test(line))) return value;
+  if (!lines.some(line => /^\s*- /.test(line))) return null;
   const kept = [];
   for (const line of lines) {
     const trimmed = line.trim();
@@ -89,9 +102,163 @@ export function compactAriaSnapshot(text) {
     const role = (body.match(/^([a-z]+)/) || [])[1] || '';
     const named = /^[a-z]+ "[^"]*"/.test(body);
     const inlineText = /^[a-z]+(?: "[^"]*")?(?: \[[^\]]*\])*: \S/.test(body);
-    if (INTERACTIVE_ROLES.has(role) || named || inlineText) kept.push(`- ${body}`);
+    if (INTERACTIVE_ROLES.has(role) || named || inlineText) {
+      kept.push({ depth: line.length - line.trimStart().length, line: `- ${body}`, body, role });
+    }
+  }
+  return kept;
+}
+
+/** Roles a player types into or sets, kept first when an observation has to be cut. */
+const INPUT_ROLES = new Set(['textbox', 'searchbox', 'combobox', 'checkbox', 'radio', 'slider', 'spinbutton', 'switch', 'listbox']);
+
+/** The share of interactive nodes, counted from the end of the page, kept ahead of the middle when a page is cut. */
+export const TAIL_FRACTION = 0.1;
+
+/** The tail window is never smaller than this many interactive nodes. */
+const TAIL_MIN = 3;
+
+/** A run of this many look-alike siblings or more is collapsed when the page is over the cap. */
+export const COLLAPSE_MIN_RUN = 4;
+
+/** How many of a collapsed run are still shown, so the model sees what the items look like. */
+export const COLLAPSE_KEEP = 2;
+
+/**
+ * The shape of one node, without the words that make it unique: its role,
+ * its state attributes, whether it has a name, whether the name carries
+ * digits and how many separators. Two school cards share a shape; the card
+ * list and a "TAKE THE JOB" button after it do not.
+ *
+ * @param {string} body
+ * @returns {string}
+ */
+function nodeShape(body) {
+  const role = (body.match(/^([a-z]+)/) || [])[1] || '';
+  const name = (body.match(/^[a-z]+ "([^"]*)"/) || [])[1];
+  const attributes = [...body.matchAll(/\[([a-z-]+)(?:=[^\]]*)?\]/g)].map(match => match[1]).filter(key => key !== 'ref').join(',');
+  const nameShape = name === undefined ? '' : `n${/\d/.test(name) ? 'd' : ''}${(name.match(/[·•|/]/g) || []).length}`;
+  const inline = /: \S/.test(body.replace(/^[a-z]+(?: "[^"]*")?/, '')) ? 't' : '';
+  return `${role}|${nameShape}|${attributes}|${inline}`;
+}
+
+/**
+ * Collapse runs of look-alike siblings (a node and its kept descendants,
+ * compared by shape) into the first {@link COLLAPSE_KEEP} plus one line that
+ * says how many more there are.
+ *
+ * @param {Array<{depth: number, line: string, body: string, role: string}>} nodes
+ * @returns {{entries: Array<{line: string, role: string, marker?: boolean}>, collapsed: number}}
+ */
+export function collapseRepeatedSiblings(nodes) {
+  const unitEnd = start => {
+    let end = start + 1;
+    while (end < nodes.length && nodes[end].depth > nodes[start].depth) end += 1;
+    return end;
+  };
+  const signature = (start, end) => nodes.slice(start, end).map(node => `${node.depth - nodes[start].depth}:${nodeShape(node.body)}`).join(';');
+  const entries = [];
+  let collapsed = 0;
+  let i = 0;
+  while (i < nodes.length) {
+    const units = [[i, unitEnd(i)]];
+    const sig = signature(i, units[0][1]);
+    let next = units[0][1];
+    while (next < nodes.length && nodes[next].depth === nodes[i].depth) {
+      const end = unitEnd(next);
+      if (signature(next, end) !== sig) break;
+      units.push([next, end]);
+      next = end;
+    }
+    if (units.length >= COLLAPSE_MIN_RUN) {
+      for (const [start, end] of units.slice(0, COLLAPSE_KEEP)) {
+        for (const node of nodes.slice(start, end)) entries.push({ line: node.line, role: node.role });
+      }
+      const more = units.length - COLLAPSE_KEEP;
+      collapsed += more;
+      entries.push({ line: `- ... and ${more} more like it (${nodes[i].role || 'items'}; target one by its name)`, role: '', marker: true });
+      i = next;
+    } else {
+      entries.push({ line: nodes[i].line, role: nodes[i].role });
+      i += 1;
+    }
   }
-  return kept.join('\n');
+  return { entries, collapsed };
+}
+
+/**
+ * Fit a compacted aria snapshot under the web ceiling without losing the
+ * part of the page the persona has to act on.
+ *
+ * Under the cap nothing changes. Over it, in this order: interactive nodes
+ * are what is kept (the last tenth of them on the page first, since that is
+ * where a form after a long list sits, then inputs, then buttons and links,
+ * then any role with a name); runs of look-alike siblings collapse to their
+ * first two and a count; nodes that are new since the previous observation
+ * come next; then
+ * the rest, and only what still does not fit is trimmed. Kept lines stay in
+ * page order, and a first line says how many items were collapsed and how
+ * many lines were left out, so the model knows the screen was cut.
+ *
+ * @param {string} text the raw aria snapshot
+ * @param {{max?: number, previous?: Set<string>|null}} [options]
+ * @returns {{text: string, lines: Array<string>}|null} null when the text is not an aria snapshot
+ */
+export function fitAriaSnapshot(text, { max = WEB_OBSERVATION_MAX_CHARS, previous = null } = {}) {
+  const nodes = compactAriaNodes(text);
+  if (nodes === null) return null;
+  const lines = nodes.map(node => node.line);
+  const full = lines.join('\n');
+  if (full.length <= max) return { text: full, lines };
+
+  const { entries, collapsed } = collapseRepeatedSiblings(nodes);
+  const header = (left, out) => {
+    const parts = [];
+    if (left > 0) parts.push(`${left} similar items collapsed`);
+    if (out > 0) parts.push(`${out} lines left out to fit`);
+    return parts.length > 0 ? `[${parts.join('; ')}]` : '';
+  };
+  const collapsedText = entries.map(entry => entry.line).join('\n');
+  const withHeader = `${header(collapsed, 0)}\n${collapsedText}`;
+  if (withHeader.length <= max) return { text: withHeader, lines };
+
+  const rank = entry => {
+    if (entry.marker) return 0;
+    if (INPUT_ROLES.has(entry.role)) return 1;
+    if (entry.role === 'button' || entry.role === 'link') return 2;
+    if (INTERACTIVE_ROLES.has(entry.role) || /^- [a-z]+ "/.test(entry.line)) return 3;
+    return 5;
+  };
+  const isNew = entry => Boolean(previous) && !entry.marker && !previous.has(entry.line);
+  // The last interactive nodes on the page are usually its form or its
+  // confirm button (a panel after a list). When the list could not be
+  // collapsed they would otherwise lose to the list on page order, so the
+  // tail window, the final tenth of interactive nodes, ranks above the middle.
+  const interactive = entries.map((entry, index) => ({ entry, index })).filter(({ entry }) => !entry.marker && rank(entry) < 5);
+  const tailSize = Math.max(TAIL_MIN, Math.ceil(interactive.length * TAIL_FRACTION));
+  const tail = new Set(interactive.slice(-tailSize).map(({ index }) => index));
+  const tierOf = (entry, index) => {
+    if (entry.marker) return 0;
+    if (tail.has(index)) return 1 + (isNew(entry) ? 0 : 0.5);
+    if (rank(entry) < 5) return 2 + rank(entry) * 2 + (isNew(entry) ? 0 : 1);
+    return isNew(entry) ? 20 : 21;
+  };
+  const order = entries
+    .map((entry, index) => ({ entry, index, tier: tierOf(entry, index) }))
+    .sort((a, b) => a.tier - b.tier || a.index - b.index);
+  // Room for the header line, whose figures are not known until the end.
+  const budget = max - 80;
+  const chosen = new Set();
+  let used = 0;
+  for (const candidate of order) {
+    const cost = candidate.entry.line.length + 1;
+    if (used + cost > budget) continue;
+    chosen.add(candidate.index);
+    used += cost;
+  }
+  const kept = entries.filter((entry, index) => chosen.has(index)).map(entry => entry.line);
+  const out = entries.length - kept.length;
+  return { text: truncateObservation(`${header(collapsed, out)}\n${kept.join('\n')}`, max), lines };
 }
 
 /**
@@ -323,6 +490,8 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
   let shots = 0;
   let lastHash = null;
   let lastObservedHash = null;
+  /** The compacted lines of the last observation, so a cut can prefer what is new. */
+  let lastObservedLines = null;
   const consoleBuffer = [];
 
   /** @returns {any} */
@@ -367,13 +536,33 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
   }
 
   /**
-   * What the model is shown for a raw snapshot. `observe` and `act` both use
-   * this, so the two hashes compare like with like.
+   * What the model is shown for a raw snapshot, and the lines it was built
+   * from. A page over the web ceiling is fitted rather than cut at the
+   * character limit (see fitAriaSnapshot), so a panel after a long list
+   * still reaches the model.
    * @param {{text: string, aria: boolean}} snap
-   * @returns {string}
+   * @returns {{text: string, lines: Array<string>|null}}
    */
   function shownText(snap) {
-    return truncateObservation(snap.aria ? compactAriaSnapshot(snap.text) : snap.text, WEB_OBSERVATION_MAX_CHARS);
+    if (snap.aria) {
+      const fitted = fitAriaSnapshot(snap.text, { max: WEB_OBSERVATION_MAX_CHARS, previous: lastObservedLines ? new Set(lastObservedLines) : null });
+      if (fitted) return fitted;
+    }
+    return { text: truncateObservation(snap.text, WEB_OBSERVATION_MAX_CHARS), lines: null };
+  }
+
+  /**
+   * What `changed` and `unchanged` compare. For an aria snapshot that is the
+   * whole compacted tree, not the fitted text: the fit depends on the
+   * previous observation, so hashing it would read the same screen as
+   * changed. For the innerText fallback it is the truncated text, so a
+   * difference the model never saw is not a change.
+   * @param {{text: string, aria: boolean}} snap
+   * @param {{text: string, lines: Array<string>|null}} shown
+   * @returns {string}
+   */
+  function hashText(snap, shown) {
+    return shown.lines ? shown.lines.join('\n') : shown.text;
   }
 
   /**
@@ -468,9 +657,10 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
       step += 1;
       const snap = await snapshotText();
       const text = snap.text;
+      const shown = shownText(snap);
       const observation = {
         step,
-        text: shownText(snap),
+        text: shown.text,
         url: current.url(),
         elapsedMs: Date.now() - started
       };
@@ -497,11 +687,11 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
           // read it as a blank screen, which is what it is.
         }
       }
-      // Hash what was REPORTED, not the untruncated snapshot: `act` compares
-      // against this, so two screens that differ only past the truncation
-      // point would otherwise read as changed to `act` and identical to
-      // anything reading the observation.
-      lastHash = observationHash(observation);
+      // Hash what hashText says, which `act` hashes the same way: the whole
+      // compacted tree for an aria snapshot (a change anywhere in it is shown,
+      // because a fit keeps new nodes first), the truncated text otherwise.
+      lastHash = observationHash({ text: hashText(snap, shown), screenshotBase64: observation.screenshotBase64 });
+      lastObservedLines = shown.lines;
       // The same screen twice is sent as a marker rather than in full
       // (./tools.js), which is most of what a stuck persona would otherwise
       // pay for.
@@ -554,10 +744,10 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
 
       let changed = false;
       try {
-        // The same truncation observe() applies, so the two hashes are
-        // comparable. Without it a long screen is always "changed".
+        // The same basis observe() hashes, so the two are comparable.
+        // Without it a long screen is always "changed".
         const snap = await snapshotText();
-        const after = { text: shownText(snap) };
+        const after = { text: hashText(snap, shownText(snap)) };
         if (String(snap.text).trim().length < THIN_OBSERVATION_CHARS) {
           const frame = await current.screenshot({ type: 'png', timeout: ACTION_TIMEOUT_MS });
           after.screenshotBase64 = frame.toString('base64');

package/src/run/heartbeat.js

@@ -58,6 +58,11 @@ export function createHeartbeat({
   let canceled = false;
   let failures = 0;
   let inFlight = false;
+  // Set by stop(). A beat already on the wire when the run ends must not
+  // write its progress after the run's final state: in 0.2.8 one landed
+  // after the outcome and put a finished run back to `playing` in the
+  // journal.
+  let stopped = false;
   // The progress the loop last reported. The interval beat carries it too,
   // rather than resetting the server's and the journal's idea of the
   // checkpoint to zero every five seconds.
@@ -78,7 +83,7 @@ export function createHeartbeat({
     try {
       const answer = await api.runs.heartbeat(gameId, jobId, runId, { heartbeat: true, ...progress });
       failures = 0;
-      if (onBeat) await onBeat({ ...progress });
+      if (onBeat && !stopped) await onBeat({ ...progress });
       if (answer && answer.run && answer.run.usage && onUsage) onUsage(answer.run.usage);
       if (answer && answer.cancel_requested && !canceled) {
         canceled = true;
@@ -111,6 +116,7 @@ export function createHeartbeat({
     },
 
     start() {
+      stopped = false;
       if (timer) return this;
       timer = setIntervalImpl(() => {
         beat().catch(() => {});
@@ -121,6 +127,7 @@ export function createHeartbeat({
     },
 
     stop() {
+      stopped = true;
       if (timer) clearIntervalImpl(timer);
       timer = null;
       return this;

package/src/run/personaLoop.js

@@ -2,11 +2,10 @@ import { selectAction, JEV_CONTROLLER_PROMPT } from './actionSelector.js';
 import { mkdir, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import { prepareMessages, MAX_IMAGES_IN_CONTEXT, KEEP_OBSERVATIONS, shrinkImages } from './context.js';
-import { observationHash } from './drivers/driver.js';
 import { withRollingCacheBreakpoint } from './fixLoop.js';
 import { callStepWithRetry, ProxyError, STEPS } from './model.js';
 import { RUN_FILES } from './paths.js';
-import { checkEvidence, createReportValidator, lintMarkdownSections, repairInstruction, REQUIRED_MARKDOWN_SECTIONS, stampReport, writeReportFiles } from './report.js';
+import { checkEvidence, createReportValidator, lintMarkdownSections, normaliseEvidence, repairInstruction, REQUIRED_MARKDOWN_SECTIONS, stampReport, writeReportFiles } from './report.js';
 import { assertNoSecrets } from './secretScan.js';
 import { ACTION_TOOLS, PLAY_TOOLS, SWIPE_TOOL, REPORT_TOOLS, executeTool } from './tools.js';
 import { createTranscript } from './transcript.js';
@@ -388,7 +387,7 @@ async function playLoop({
   latestObservation = first.text;
   visualObservation = Boolean(first.screenshotBase64);
   transcript.write({ kind: 'observation', step: first.step, text: first.text, url: first.url });
-  const landing = await safeScreenshot(driver, 'landing', screenshots, transcript);
+  const landing = await safeScreenshot(driver, 'landing', screenshots, transcript, observations.length);
   messages.push({
     role: 'user',
     content: [
@@ -503,7 +502,7 @@ async function playLoop({
           transcript.write({ kind: 'note', tool: toolUse.name, note });
         },
         onScreenshot: shot => {
-          screenshots.push({ ...shot, order: nextOrder() });
+          screenshots.push({ ...shot, observation: observations.length, order: nextOrder() });
           transcript.write({ kind: 'screenshot', path: shot.relativePath, reason: shot.reason });
         }
       });
@@ -552,7 +551,7 @@ async function playLoop({
       // with the persona's view of the screen in front of it. A 4xx on a
       // resource is a `network` note, which the report step reads as context
       // rather than as a bug (a missing favicon is not a broken game).
-      const shot = await safeScreenshot(driver, 'console-error', screenshots, transcript);
+      const shot = await safeScreenshot(driver, 'console-error', screenshots, transcript, observations.length);
       for (const entry of consoleErrors) {
         const kind = RESOURCE_4XX.test(String(entry.text)) ? 'network' : 'bug';
         const note = { kind, text: `${entry.type}: ${entry.text}`, automatic: true, ...(shot ? { screenshot: shot.relativePath } : {}) };
@@ -592,7 +591,7 @@ async function playLoop({
     quitDetail = `the action budget of ${maxActions} ran out`;
   }
 
-  const final = await safeScreenshot(driver, 'final', screenshots, transcript);
+  const final = await safeScreenshot(driver, 'final', screenshots, transcript, observations.length);
   return {
     actionsTaken: actions,
     turns,
@@ -651,12 +650,15 @@ async function reportPhase({
         observationLines(observations),
         '',
         'Each note carries "after": the URL and title of the observation that followed it. A blocker or major finding must cite the observation index where the failure is visible, as evidence {"type": "transcript_step", "ref": "observation N"}, and must not be contradicted by its note\'s "after" state or by a later persona note: if the next screen or a later note shows the game working, it is not a blocker or a major. Cite a note as {"type": "transcript_step", "ref": "note N"}.',
+        'A blocker or major finding must also cite the screenshot taken at or after that observation (the list below says which observation each one followed), and that screenshot must not contradict it: if it shows the thing the finding says is missing or broken, such as the field or the button, it is not a blocker or a major. A text observation can be cut short on a long page; the screenshot is what the player saw.',
+        '',
+        'Every evidence entry is an object with exactly two keys, "type" and "ref", and nothing else. A screenshot is {"type": "screenshot", "ref": "screenshots/04-example.png"}, never "path" or "file".',
         '',
         'Never write calendar dates or times in the prose of report.md or report.json. The runner stamps when the report was generated, so leave the template\'s Date out rather than writing a placeholder for it.',
         'Replace every <...> slot in the template with real content. Never copy a <...> placeholder into the report.',
         '',
         'Screenshots available as evidence (use these exact paths in evidence refs):',
-        screenshots.length > 0 ? screenshots.map(shot => `- ${shot.relativePath}${shot.reason ? ` (${shot.reason})` : ''}`).join('\n') : '(none)',
+        screenshots.length > 0 ? screenshots.map(screenshotLine).join('\n') : '(none)',
         '',
         `report.md must contain these section headings, spelled exactly: ${REQUIRED_MARKDOWN_SECTIONS.map(section => `"${section}"`).join(', ')}.`,
         'Write report.md first, then report.json. Leave usage as zeros; the runner fills it in.'
@@ -667,12 +669,20 @@ async function reportPhase({
   let report = null;
   let markdown = null;
   let lint = [];
-
-  // Two attempts at most: the first, and one repair if the schema or the
-  // evidence refs refuse it. Each attempt is a metered question, and the
-  // dogfood gate asks for "first or the repair attempt", not an
-  // open ended loop.
-  for (let attempt = 1; attempt <= 2; attempt += 1) {
+  // The newest report.json the model wrote, across turns: a repair turn that
+  // rewrites only report.md keeps the report.json it already wrote.
+  let jsonText = null;
+  let repaired = false;
+  let continued = false;
+
+  // One repair at most when the schema or the evidence refs refuse what was
+  // written: each turn is a metered question, and the dogfood gate asks for
+  // "first or the repair attempt", not an open ended loop. A turn that wrote
+  // report.md and stopped before report.json is not a refused report, so it
+  // gets one plain "now write report.json" that does not spend the repair.
+  // In 0.2.8 it did, and the report.json that came back had its five
+  // screenshot refs under "path" with no turn left to fix them.
+  for (let attempt = 1; ; attempt += 1) {
     const answer = await callStepWithRetry({
       client,
       jobId,
@@ -695,12 +705,13 @@ async function reportPhase({
       files.set(name, String(toolUse.input.content || ''));
     }
     if (files.has(RUN_FILES.reportMd)) markdown = files.get(RUN_FILES.reportMd);
+    if (files.has(RUN_FILES.reportJson)) jsonText = files.get(RUN_FILES.reportJson);
 
     let parsed = null;
     let parseError = null;
-    if (files.has(RUN_FILES.reportJson)) {
+    if (jsonText !== null) {
       try {
-        parsed = JSON.parse(files.get(RUN_FILES.reportJson));
+        parsed = JSON.parse(jsonText);
       } catch (error) {
         parseError = String(error && error.message);
       }
@@ -720,7 +731,13 @@ async function reportPhase({
       quitReason: play.quitReason,
       quitDetail: play.quitDetail
     };
-    const candidate = parsed ? capNetworkOnlyFindings(stampReport(parsed, facts), { notes, screenshots }) : null;
+    // An evidence path under the wrong key is moved to `ref` before anything
+    // judges the report; see normaliseEvidence.
+    const normalised = parsed ? normaliseEvidence(parsed, runDirectory) : null;
+    if (normalised && normalised.changed > 0) {
+      transcript.write({ kind: 'report_evidence_normalised', attempt, entries: normalised.changed });
+    }
+    const candidate = normalised ? capNetworkOnlyFindings(stampReport(normalised.report, facts), { notes, screenshots }) : null;
     const validation = candidate ? validator.validate(candidate) : { ok: false, details: [{ instancePath: '/', message: parseError || 'report.json was not written' }] };
     const evidenceProblems = candidate ? await checkEvidence(candidate, runDirectory) : [];
     // The server lints the same four sections at `complete` and records the
@@ -742,24 +759,39 @@ async function reportPhase({
       ...evidenceProblems,
       ...missingSections.map(section => `report.md has no "${section}" section`)
     ];
-    if (attempt === 2) {
+
+    // Every tool_use the next request echoes back needs its tool_result, or
+    // the API refuses the request. A write cut off at the token limit is
+    // malformed: it was not saved, and it is answered so.
+    const toolResults = answer.toolUses.map(toolUse => (toolUse.malformed
+      ? { type: 'tool_result', tool_use_id: toolUse.id, is_error: true, content: [{ type: 'text', text: CUT_OFF_WRITE }] }
+      : { type: 'tool_result', tool_use_id: toolUse.id, content: [{ type: 'text', text: 'saved' }] }));
+
+    if (jsonText === null && files.has(RUN_FILES.reportMd) && !continued) {
+      continued = true;
+      messages.push({ role: 'assistant', content: assistantContent(answer) });
+      messages.push({ role: 'user', content: toolResults.concat([{ type: 'text', text: CONTINUE_WITH_JSON }]) });
+      continue;
+    }
+
+    if (repaired) {
       if (!candidate) throw new Error(`the report step did not produce a valid report.json: ${lint.join('; ')}`);
-      // Second attempt still imperfect: keep it, record the lint, and let
+      // Still imperfect after the repair: keep it, record the lint, and let
       // the server have the final word. It validates the same schema and
       // drops unevidenced findings itself, so this is a report with known
-      // problems rather than no report at all.
+      // problems rather than no report at all. Every problem on its own line,
+      // before anything is uploaded, so a refusal at complete is never the
+      // first the developer hears of it.
       report = candidate;
-      log(`report.json still has problems after one repair: ${lint.join('; ')}`);
+      log(`report.json still has problems after one repair (${lint.length}); the server will refuse a report that fails its schema:\n${lint.map(line => `  - ${line}`).join('\n')}`);
       break;
     }
 
+    repaired = true;
     messages.push({ role: 'assistant', content: assistantContent(answer) });
     messages.push({
       role: 'user',
-      content: answer.toolUses
-        .filter(toolUse => !toolUse.malformed)
-        .map(toolUse => ({ type: 'tool_result', tool_use_id: toolUse.id, content: [{ type: 'text', text: 'saved' }] }))
-        .concat([{ type: 'text', text: repairInstruction(validation.details, evidenceProblems, missingSections) }])
+      content: toolResults.concat([{ type: 'text', text: repairInstruction(validation.details, evidenceProblems, missingSections) }])
     });
   }
 
@@ -901,10 +933,10 @@ function redactToolInput(toolUse) {
   return rest;
 }
 
-async function safeScreenshot(driver, name, screenshots, transcript) {
+async function safeScreenshot(driver, name, screenshots, transcript, observation = null) {
   try {
     const shot = await driver.screenshot(name);
-    screenshots.push({ ...shot, reason: name, order: nextOrder() });
+    screenshots.push({ ...shot, reason: name, ...(Number.isInteger(observation) ? { observation } : {}), order: nextOrder() });
     transcript.write({ kind: 'screenshot', path: shot.relativePath, reason: name });
     return shot;
   } catch (error) {
@@ -1122,6 +1154,28 @@ export function noteLine(note, i) {
   return `${i + 1}. [${note.kind}${note.severity ? `/${note.severity}` : ''}] ${note.text}${tags.length > 0 ? ` (${tags.join(', ')})` : ''}${after}`;
 }
 
+/** The tool_result for a write_file call that was cut off before its input was complete. */
+export const CUT_OFF_WRITE = 'That write_file call was cut off before it finished, so nothing was saved. Write that file again, shorter if it has to be.';
+
+/** The one follow-up a report turn gets for writing report.md and stopping. */
+export const CONTINUE_WITH_JSON = 'report.md is saved. Now write report.json with write_file. Every evidence entry is exactly {"type": ..., "ref": ...}; a screenshot is {"type": "screenshot", "ref": "screenshots/04-example.png"}.';
+
+/**
+ * A screenshot as one line of the report prompt: its path, the observation
+ * it followed (what a blocker or major has to be checked against) and why
+ * it was taken.
+ *
+ * @param {Object} shot
+ * @returns {string}
+ */
+export function screenshotLine(shot) {
+  const tags = [
+    Number.isInteger(shot.observation) && shot.observation > 0 ? `after observation ${shot.observation}` : null,
+    shot.reason ? String(shot.reason) : null
+  ].filter(Boolean);
+  return `- ${shot.relativePath}${tags.length > 0 ? ` (${tags.join('; ')})` : ''}`;
+}
+
 /** The most observations listed in the report prompt; the newest are kept. */
 export const MAX_OBSERVATIONS_IN_REPORT = 200;
 

package/src/run/report.js

@@ -127,6 +127,86 @@ export function stampReport(report, facts) {
   return stamped;
 }
 
+/** Keys a model has been seen to put an evidence path under instead of `ref`. */
+const REF_ALIASES = ['path', 'file', 'screenshot', 'src', 'href', 'location', 'value'];
+
+/** The four evidence types the schema allows. */
+const EVIDENCE_TYPES = new Set(['screenshot', 'video_ts', 'transcript_step', 'file_line']);
+
+/**
+ * One evidence entry in the schema's shape, `{type, ref}` and nothing else,
+ * when the model's own words say unambiguously what it meant.
+ *
+ * The 0.2.8 report that failed upload wrote
+ * `{"type": "screenshot", "path": "screenshots/04-04-school-selected.png"}`
+ * five times: a real file, under the wrong key. The schema refuses the extra
+ * key and the missing `ref`, so the whole report was refused and the run
+ * failed over a key name. This moves the value, never invents one: a path
+ * still has to name a file in the run directory (`checkEvidence`), and the
+ * server still checks it against what was uploaded.
+ *
+ * Also: a bare string becomes a screenshot or a transcript step by its look;
+ * a missing type on a `screenshots/...` ref becomes `screenshot`; a ref
+ * written `./screenshots/x.png` or as an absolute path inside the run
+ * directory becomes the run relative path the server's manifest uses.
+ *
+ * @param {any} item
+ * @param {string|null} runDirectory
+ * @returns {any} the normalised entry, or the input unchanged when it is not an object it can read
+ */
+export function normaliseEvidenceItem(item, runDirectory = null) {
+  if (typeof item === 'string') {
+    const text = item.trim();
+    if (/^(?:\.\/)?screenshots\//.test(text)) return normaliseEvidenceItem({ type: 'screenshot', ref: text }, runDirectory);
+    if (/^(?:observation|note|step|action)\s*#?\s*\d+$/i.test(text)) return { type: 'transcript_step', ref: text };
+    return item;
+  }
+  if (!item || typeof item !== 'object' || Array.isArray(item)) return item;
+  let ref = typeof item.ref === 'string' && item.ref.trim() ? item.ref : null;
+  if (ref === null) {
+    const alias = REF_ALIASES.find(key => typeof item[key] === 'string' && item[key].trim());
+    if (alias) ref = item[alias];
+  }
+  let type = typeof item.type === 'string' ? item.type : null;
+  if (!type && ref && /^(?:\.\/)?screenshots\//.test(ref.trim())) type = 'screenshot';
+  if (type === null || ref === null || !EVIDENCE_TYPES.has(type)) return item;
+  ref = ref.trim();
+  if (type === 'screenshot') ref = runRelative(ref, runDirectory);
+  return { type, ref };
+}
+
+function runRelative(ref, runDirectory) {
+  let value = ref.replace(/\\/g, '/');
+  if (runDirectory && path.isAbsolute(ref)) {
+    const relative = path.relative(path.resolve(runDirectory), path.resolve(ref));
+    if (relative && !relative.startsWith('..') && !path.isAbsolute(relative)) value = relative.split(path.sep).join('/');
+  }
+  while (value.startsWith('./')) value = value.slice(2);
+  return value;
+}
+
+/**
+ * Every finding's evidence through {@link normaliseEvidenceItem}.
+ *
+ * @param {Object} report
+ * @param {string|null} [runDirectory]
+ * @returns {{report: Object, changed: number}} a new report, and how many entries were rewritten
+ */
+export function normaliseEvidence(report, runDirectory = null) {
+  if (!report || typeof report !== 'object' || !Array.isArray(report.findings)) return { report, changed: 0 };
+  let changed = 0;
+  const findings = report.findings.map(finding => {
+    if (!finding || typeof finding !== 'object' || !Array.isArray(finding.evidence)) return finding;
+    const evidence = finding.evidence.map(item => {
+      const out = normaliseEvidenceItem(item, runDirectory);
+      if (JSON.stringify(out) !== JSON.stringify(item)) changed += 1;
+      return out;
+    });
+    return { ...finding, evidence };
+  });
+  return { report: { ...report, findings }, changed };
+}
+
 /**
  * Check every `screenshot` evidence ref against the files actually on disk.
  *
@@ -151,8 +231,17 @@ export async function checkEvidence(report, runDirectory) {
       continue;
     }
     for (const item of evidence) {
-      if (!item || item.type !== 'screenshot') continue;
-      const ref = String(item.ref || '');
+      if (!item || typeof item !== 'object') {
+        problems.push(`${finding.id}: an evidence entry is not an object; write {"type": ..., "ref": ...}`);
+        continue;
+      }
+      if (typeof item.ref !== 'string' || !item.ref.trim()) {
+        const keys = Object.keys(item).filter(key => key !== 'type');
+        problems.push(`${finding.id}: a ${item.type || 'typeless'} evidence entry has no "ref"${keys.length > 0 ? ` (it has ${keys.map(key => `"${key}"`).join(', ')})` : ''}; every entry is exactly {"type": ..., "ref": ...}`);
+        continue;
+      }
+      if (item.type !== 'screenshot') continue;
+      const ref = String(item.ref);
       const candidate = path.resolve(runDirectory, ref);
       // Refuse a ref that climbs out of the run directory before touching
       // the filesystem with it.
@@ -214,6 +303,8 @@ export default {
   createReportValidator,
   stampReport,
   checkEvidence,
+  normaliseEvidence,
+  normaliseEvidenceItem,
   writeReportFiles,
   repairInstruction,
   lintMarkdownSections,

package/src/run/resume.js

@@ -62,6 +62,15 @@ export async function resumeJob({ api, gameId, jobId, job, runs, state, repoRoot
     return EXIT.ENVIRONMENT;
   }
 
+  // A run whose report the server refused is done here, whatever the server
+  // says its state is: it is never finalised from its transcript (which would
+  // rewrite the kept report.json and report.md) and never replayed.
+  for (const [runId, entry] of Object.entries(journal.runs || {})) {
+    if (entry && entry.report_refused === true) {
+      log(`run ${runId}: the server refused its report, so it is not replayed or rebuilt. Its report.json, report.md and screenshots are in ${runDirFor(repoRoot, jobId, runId)}.`);
+    }
+  }
+
   // Reconcile the journal with what the server actually thinks: a run the sweep
   // marked interrupted, or one that finished after the journal was last
   // written, is the server's word against a stale file, and the server wins.

package/src/run/state.js

@@ -192,7 +192,10 @@ export async function patchRun(repoRoot, state, runId, patch) {
  * @returns {boolean}
  */
 export function isRunDone(entry) {
-  return Boolean(entry) && (entry.completed === true || RUN_DONE_STATES.includes(entry.state));
+  // A run whose report the server refused was played to the end: replaying
+  // it would spend a unit again, and rebuilding its report from the
+  // transcript would overwrite the one the developer is meant to read.
+  return Boolean(entry) && (entry.completed === true || entry.report_refused === true || RUN_DONE_STATES.includes(entry.state));
 }
 
 export default { SCHEMA_VERSION, newState, newRunEntry, save, load, patchRun, isRunDone, stateFile };