ravensight-playtest 0.5.2 → 0.7.0

package/src/run/liveWork.js

@@ -0,0 +1,103 @@
+import { rmSync } from 'node:fs';
+
+/**
+ * What this process has running or lying around right now: Godot processes and the
+ * temp copies they run on. Each is removed by its owner in the ordinary course of
+ * things. This list is for the rest: a second Ctrl-C, a SIGTERM, a `process.exit`.
+ * Death by a signal runs no `finally` and no exit handler, and a check's process group
+ * does not hear the terminal's Ctrl-C at all, so without this a killed CLI leaves a
+ * Godot running and a copy of the project in the temp directory.
+ */
+const live = new Set();
+const SIGNALS = ['SIGINT', 'SIGTERM', 'SIGHUP'];
+let installed = false;
+
+function onExit() {
+  finishAll();
+}
+
+let claimed = false;
+
+/**
+ * A handler has taken this signal, even if it has already stepped aside. A listener
+ * that removes itself during the signal is still the one that answered it, and the
+ * count below cannot see that.
+ */
+export function claimSignal() {
+  claimed = true;
+  setImmediate(() => { claimed = false; });
+}
+
+/** A signal nobody else is handling: tidy up, then die of it as the shell expects. */
+function onSignal(signal) {
+  if (claimed || process.listenerCount(signal) > 1) return;
+  dieOf(signal);
+}
+
+/**
+ * @param {string} signal
+ * @param {Object} [target] test seam
+ */
+export function dieOf(signal, target = process) {
+  finishAll();
+  for (const name of SIGNALS) process.removeListener(name, onSignal);
+  target.kill(target.pid, signal);
+}
+
+function install() {
+  if (installed) return;
+  installed = true;
+  process.on('exit', onExit);
+  for (const name of SIGNALS) process.on(name, onSignal);
+}
+
+function uninstall() {
+  if (!installed || live.size > 0) return;
+  installed = false;
+  process.removeListener('exit', onExit);
+  for (const name of SIGNALS) process.removeListener(name, onSignal);
+}
+
+/**
+ * @param {{stop?: () => void, kill?: () => void, root?: string|null|(() => Array<string>)}} entry `stop` also runs on
+ * a first Ctrl-C and `kill` only when the process is going away; `root` is what to remove then
+ * @returns {() => void} call when the work is over
+ */
+export function track(entry) {
+  live.add(entry);
+  install();
+  return () => {
+    live.delete(entry);
+    uninstall();
+  };
+}
+
+/** Stop every process, leaving their temp copies for their owners to remove. */
+export function stopAll() {
+  for (const entry of live) {
+    try {
+      entry.stop?.();
+    } catch {
+      // Already gone.
+    }
+  }
+}
+
+/** Stop every process and remove every temp copy, synchronously. */
+export function finishAll() {
+  stopAll();
+  for (const entry of live) {
+    try {
+      entry.kill?.();
+      const roots = typeof entry.root === 'function' ? entry.root() : [entry.root];
+      for (const root of roots) {
+        if (root) rmSync(root, { recursive: true, force: true });
+      }
+    } catch {
+      // Nothing more can be done from here.
+    }
+  }
+  live.clear();
+}
+
+export default { track, stopAll, finishAll, dieOf, claimSignal };

package/src/run/model.js

@@ -54,7 +54,15 @@ export const STEPS = Object.freeze({
   // allowlist includes `write_file`.
   fixPlan: 'fix.plan',
   fixApply: 'fix.apply',
-  fixSummarize: 'fix.summarize'
+  fixSummarize: 'fix.summarize',
+  // The `feature` module's three (Labs beta), spendable only from a run whose
+  // module is `feature`. `feature.apply` has the same tools as `fix.apply`.
+  featurePlan: 'feature.plan',
+  featureApply: 'feature.apply',
+  featureSummarize: 'feature.summarize',
+  // The `prototype` module's one (Labs, Experimental): the game design. Its only
+  // tools answer the design or decline the brief, so it cannot touch anything.
+  prototypeDesign: 'prototype.design'
 });
 
 /** Default retry budget for the answers that are worth retrying. */
@@ -63,6 +71,9 @@ export const MAX_ATTEMPTS = 4;
 /** Codes the proxy raises that a retry can actually clear. */
 const RETRYABLE_CODES = new Set(['concurrency_limited', 'velocity_limited', 'rate_limited', 'upstream_error']);
 
+/** What the SDK calls an answer that never came back. */
+const CONNECTION_ERRORS = new Set(['APIConnectionError', 'APIConnectionTimeoutError']);
+
 export class ProxyError extends Error {
   /**
    * @param {{status: number, code: string, message: string, retryAfter?: number|null, figures?: Object}} init
@@ -92,7 +103,11 @@ export class ProxyError extends Error {
   }
 
   get isRetryable() {
-    if (this.status === 502) return true;
+    if (this.status === 502 || this.status === 503 || this.status === 504) return true;
+    // No status at all: the answer never arrived. A lost call costs the run's model
+    // budget at most, never a charge, and one dropped connection should not end a
+    // job the developer paid for. Any other statusless error is a defect here.
+    if (this.status === 0) return CONNECTION_ERRORS.has(this.code);
     return this.status === 429 && RETRYABLE_CODES.has(this.code);
   }
 }
@@ -330,6 +345,11 @@ export async function callStep({
         toolUses.push({ id: block.id, name: block.name, input: null, malformed: true, raw: json });
         continue;
       }
+      // The same for JSON that parses into something no tool takes.
+      if (input === null || typeof input !== 'object' || Array.isArray(input)) {
+        toolUses.push({ id: block.id, name: block.name, input: null, malformed: true, raw: json });
+        continue;
+      }
     }
     toolUses.push({ id: block.id, name: block.name, input, malformed: false });
   }
@@ -337,6 +357,68 @@ export async function callStep({
   return { text, toolUses, stopReason, usage, model, tier, budget };
 }
 
+/**
+ * THE HOURLY SPEND HOLD. The proxy refuses a call once a studio has spent its
+ * hourly model limit, and that refusal costs nothing. A large feature's budget is
+ * bigger than one hour's limit, so this is an ordinary thing to meet on a job the
+ * customer paid for, and failing the job over it would throw away work that one
+ * wait would save. So the call waits, says how long, and carries on, and the
+ * attempt count does not move.
+ *
+ * Bounded: at most `MAX_SPEND_HOLDS` hours in one call, because a studio whose own
+ * parallel work keeps refilling the window should hear about it rather than wait
+ * forever. Stoppable: `shouldStop` is checked every `SPEND_HOLD_POLL_MS`, which is
+ * also how often the call is re-asked, so a cap that clears early resumes early.
+ * The limit itself is the server's and nothing here can change it. The job's
+ * heartbeat keeps running during a hold, so a paused run is not read as a dead one.
+ */
+export const MAX_SPEND_HOLDS = 4;
+export const SPEND_HOLD_POLL_MS = 60000;
+
+const spendHold = { log: message => process.stderr.write(`${message}\n`), shouldStop: () => false, pollMs: null };
+
+/**
+ * Who hears about a hold, and what stops one. Set by the command that owns the
+ * interrupt; a call with neither still pauses and prints to stderr.
+ *
+ * @param {{log?: (message: string) => void, shouldStop?: () => boolean, pollMs?: number|null}} [options]
+ */
+export function configureSpendHold({ log, shouldStop, pollMs } = {}) {
+  if (log) spendHold.log = log;
+  if (shouldStop) spendHold.shouldStop = shouldStop;
+  if (pollMs !== undefined) spendHold.pollMs = pollMs;
+}
+
+/** The poll interval: this module's, or the local harness's by environment. */
+function holdPollMs() {
+  if (spendHold.pollMs) return spendHold.pollMs;
+  const fromEnv = Number(process.env.RAVENSIGHT_SPEND_HOLD_POLL_MS);
+  return Number.isFinite(fromEnv) && fromEnv > 0 ? fromEnv : SPEND_HOLD_POLL_MS;
+}
+
+/** Plain words for how long a hold is. */
+export function holdWords(seconds) {
+  if (seconds < 60) return `${Math.max(1, Math.round(seconds))} seconds`;
+  const minutes = Math.ceil(seconds / 60);
+  return minutes === 1 ? 'about a minute' : `about ${minutes} minutes`;
+}
+
+/**
+ * Wait out one hold, re-asking each poll. Answers false when told to stop.
+ *
+ * @param {ProxyError} error
+ * @param {Function} [sleep]
+ * @returns {Promise<boolean>}
+ */
+async function holdFor(error, sleep) {
+  const seconds = Number(error.figures.hold_seconds);
+  const limit = Number(error.figures.limit_usd);
+  const at = typeof error.figures.resets_at === 'string' ? new Date(error.figures.resets_at).toISOString().slice(11, 16) : null;
+  spendHold.log(`Paused: this studio reached its hourly model spend limit${Number.isFinite(limit) ? ` of $${limit}` : ''}. Waiting ${holdWords(seconds)}${at ? `, until ${at} UTC` : ''}, then carrying on. Nothing is lost and nothing is charged for the wait. Ctrl-C stops the job and gives back what did not happen.`);
+  await wait(Math.min(seconds * 1000, holdPollMs()), sleep);
+  return !spendHold.shouldStop();
+}
+
 /**
  * @param {number} ms
  * @param {Function} [sleep]
@@ -350,8 +432,8 @@ function wait(ms, sleep) {
  * `callStep` with the retries that are worth taking, and none of the ones
  * that are not.
  *
- * Retried: 502 `upstream_error`, and a 429 whose code says the wait is the
- * fix (`concurrency_limited`, `velocity_limited`, `rate_limited`). The
+ * Retried: a 502, 503 or 504, a connection that dropped or timed out, and a 429
+ * whose code says the wait is the fix (`concurrency_limited`, `velocity_limited`, `rate_limited`). The
  * proxy's own `retry_after` is honoured when it sends one, since it knows
  * how long its hourly window or its upstream asked for.
  *
@@ -369,6 +451,9 @@ function wait(ms, sleep) {
 export async function callStepWithRetry(args) {
   const { maxAttempts = MAX_ATTEMPTS, onRetry, sleep, ...rest } = args;
   let last = null;
+  let polls = 0;
+  let holdUntil = 0;
+  let holds = 0;
   for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
     try {
       return await callStep(rest);
@@ -376,6 +461,21 @@ export async function callStepWithRetry(args) {
       if (error instanceof ProxyError && !error.isRetryable) throw error;
       if (error && (error.name === 'AbortError' || error.name === 'APIUserAbortError')) throw error;
       last = error;
+      // The spend hold is not an attempt. See above.
+      if (error instanceof ProxyError && error.code === 'velocity_limited' && Number(error.figures.hold_seconds) > 0) {
+        const now = Date.now();
+        if (now >= holdUntil) {
+          holds += 1;
+          holdUntil = now + Number(error.figures.hold_seconds) * 1000;
+          if (holds <= MAX_SPEND_HOLDS && await holdFor(error, sleep)) { attempt -= 1; continue; }
+          break;
+        }
+        polls += 1;
+        await wait(Math.min(Math.max(holdUntil - now, 1), holdPollMs()), sleep);
+        if (spendHold.shouldStop()) break;
+        attempt -= 1;
+        continue;
+      }
       if (attempt === maxAttempts) break;
       const suggested = error instanceof ProxyError && error.retryAfter ? error.retryAfter * 1000 : null;
       const backoff = suggested ?? Math.min(1000 * 2 ** (attempt - 1), 30000);
@@ -384,7 +484,7 @@ export async function callStepWithRetry(args) {
     }
   }
   const reason = last instanceof ProxyError ? `${last.status} ${last.code}: ${last.message}` : String(last && last.message);
-  throw new ModelUnavailableError(`The model proxy did not answer after ${maxAttempts} attempts (${reason}).`, last);
+  throw new ModelUnavailableError(`The model proxy did not answer after ${maxAttempts} attempts (${reason}${polls ? `, after waiting out the hourly limit` : ''}).`, last);
 }
 
 export default { createModelClient, callStep, callStepWithRetry, STEPS, ProxyError, ModelUnavailableError };

package/src/run/openPr.js

@@ -0,0 +1,107 @@
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+
+/**
+ * `fix --open-pr`: commit the fix's own files, push the branch and open a pull
+ * request with the evidence report as its body.
+ *
+ * This is the one place this CLI pushes anything, and it is fenced:
+ *
+ * - only with `--open-pr`, only after a fix the verify run scored as fixed,
+ *   and only after the developer answers yes at a prompt (`--yes` does not
+ *   count: it agrees to a price, not to publishing code);
+ * - `gh auth status` has to pass before anything is committed;
+ * - only the files the fix changed are committed, by path;
+ * - the push is a plain `git push -u origin <branch>`, never forced, and
+ *   only the branch the fix created, which must be the one checked out.
+ *
+ * Every command goes through `execFile`, never a shell.
+ */
+
+const run = promisify(execFile);
+
+/**
+ * Single quote a value for a command a person pastes into a shell. The title is
+ * the model's, so nothing inside it may expand or run.
+ *
+ * @param {*} value
+ * @returns {string}
+ */
+export function shellQuote(value) {
+  return `'${String(value).replace(/'/g, `'\\''`)}'`;
+}
+
+/**
+ * A pull request title is the model's text. One line, no control characters: a
+ * newline would turn the rest into the commit's body and its trailers.
+ *
+ * @param {*} value
+ * @returns {string}
+ */
+export function plainTitle(value) {
+  /* eslint-disable-next-line no-control-regex */
+  return String(value).replace(/[\u0000-\u001f\u007f-\u009f]+/g, match => (/[\n\r\t]/.test(match) ? ' ' : '')).replace(/ +/g, ' ').trim();
+}
+
+/**
+ * @param {Object} args
+ * @param {string} args.repoRoot
+ * @param {string} args.branch
+ * @param {Array<string>} args.paths the files the fix changed, repository relative
+ * @param {string} args.title
+ * @param {string} args.bodyFile the evidence report
+ * @param {(question: string, options: Object) => Promise<boolean>} args.confirm
+ * @param {Function} [args.exec] test seam, shaped like promisified execFile
+ * @returns {Promise<{ok: boolean, url?: string, reason?: string}>}
+ */
+export async function openPullRequest({ repoRoot, branch, paths, title: rawTitle, bodyFile, confirm, exec = run, log = () => {} }) {
+  const call = async (command, args) => {
+    try {
+      // Literal, so a file named `*` or `:(top)` commits that file. As a pathspec the
+      // first of those swept every other change in the tree into the pull request.
+      const { stdout } = await exec(command, args, { cwd: repoRoot, maxBuffer: 1024 * 1024, env: { ...process.env, GIT_LITERAL_PATHSPECS: '1' } });
+      return { ok: true, stdout: String(stdout || '') };
+    } catch (error) {
+      return { ok: false, stdout: String((error && error.stdout) || ''), stderr: String((error && error.stderr) || (error && error.message) || '') };
+    }
+  };
+  const refuse = reason => {
+    log(`${reason} Nothing was pushed.`);
+    return { ok: false, reason };
+  };
+
+  if (!Array.isArray(paths) || paths.length === 0) return refuse('There are no changed files to commit.');
+  const title = plainTitle(rawTitle);
+  const current = await call('git', ['rev-parse', '--abbrev-ref', 'HEAD']);
+  if (!current.ok || current.stdout.trim() !== branch) return refuse(`The checked out branch is not ${branch}, so --open-pr will not push it.`);
+
+  const auth = await call('gh', ['auth', 'status']);
+  if (!auth.ok) return refuse('gh is not installed or not signed in ("gh auth status" failed). Run "gh auth login" first.');
+
+  if (typeof confirm !== 'function') return refuse('--open-pr asks before it pushes, and there is no terminal to ask in.');
+  const question = [
+    `Commit ${paths.length} file(s) on ${branch}, push it to origin and open a pull request?`,
+    `  title: ${title}`,
+    ...paths.map(file => `  ${plainTitle(file)}`)
+  ].join('\n');
+  // `yes: false` on purpose: --yes agreed to the price, not to publishing.
+  const go = await confirm(question, { yes: false });
+  if (!go) return refuse('Not confirmed.');
+
+  const added = await call('git', ['add', '--', ...paths]);
+  if (!added.ok) return refuse(`git add failed: ${added.stderr.trim()}`);
+  const committed = await call('git', ['commit', '-m', title, '--only', '--', ...paths]);
+  if (!committed.ok) return refuse(`git commit failed: ${committed.stderr.trim()}`);
+  const pushed = await call('git', ['push', '-u', 'origin', branch]);
+  if (!pushed.ok) return refuse(`git push failed: ${pushed.stderr.trim()} The commit is on ${branch} locally.`);
+  const created = await call('gh', ['pr', 'create', '--head', branch, '--title', title, '--body-file', bodyFile]);
+  if (!created.ok) {
+    log(`gh pr create failed: ${created.stderr.trim()} The branch ${branch} is pushed; open the pull request by hand.`);
+    return { ok: false, reason: 'gh pr create failed' };
+  }
+  const url = created.stdout.trim().split('\n').filter(Boolean).at(-1) || '';
+  log(`Pushed ${branch} and opened ${url || 'the pull request'}.`);
+  return { ok: true, url };
+}
+
+export default { openPullRequest };

package/src/run/paths.js

@@ -66,7 +66,10 @@ export const RUN_FILES = Object.freeze({
   reportJson: 'report.json',
   usageJson: 'usage.json',
   transcript: 'transcript.jsonl',
-  video: 'session.webm'
+  video: 'session.webm',
+  // Local only (not on the upload allowlist): the screenshot evidence the
+  // runner attached to findings, so the aggregate does not cluster on it.
+  runnerEvidence: 'runner-evidence.json'
 });
 
 export const JOB_FILES = Object.freeze({

package/src/run/pendingRegister.js

@@ -0,0 +1,81 @@
+import { createHash } from 'node:crypto';
+import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+
+/**
+ * A register whose answer never arrived, kept until a later run gets one.
+ *
+ * Fix and feature run on a branch and throw their key away when a register fails, which
+ * is right for a refusal and wrong for no answer: the debit may have landed, and the only
+ * way back to that job is the same key. So the key is on disk before the request goes,
+ * and the same command finds it again and replays one job instead of buying a second.
+ */
+const directory = repoRoot => path.join(repoRoot, '.ravensight', 'pending-register');
+const fileFor = (repoRoot, { kind, gameId, target }) =>
+  path.join(directory(repoRoot), `${createHash('sha256').update(JSON.stringify([kind, gameId, target])).digest('hex').slice(0, 32)}.json`);
+
+/**
+ * Whether a failed register leaves the money unknown. A 4xx is the server saying no,
+ * except `charge_in_progress`, which means it cannot tell yet.
+ * @param {unknown} error
+ * @returns {boolean}
+ */
+export function outcomeUnknown(error) {
+  if (!error || !Number.isInteger(error.status)) return false;
+  return error.code === 'network_error' || error.code === 'charge_in_progress' || error.status === 0 || error.status >= 500;
+}
+
+/**
+ * The line that tells the developer what they are left with.
+ * @param {string} what - "this fix"
+ * @param {string} branch
+ * @returns {string}
+ */
+export function unknownOutcomeMessage(what, branch) {
+  return `The server gave no clear answer, so ${what} may have been charged. The branch ${branch} and this request's key are kept. Run the same command again: it asks about this same request, so it answers with the job if there is one and does not charge twice.`;
+}
+
+/**
+ * @param {string} repoRoot
+ * @param {{kind: string, gameId: string, target: string}} scope
+ * @returns {Promise<{idempotencyKey: string, branch: string, from: string|null, head: string}|null>}
+ */
+export async function load(repoRoot, scope) {
+  try {
+    const record = JSON.parse(await readFile(fileFor(repoRoot, scope), 'utf8'));
+    return typeof record.idempotencyKey === 'string' && typeof record.branch === 'string' ? record : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * @param {string} repoRoot
+ * @param {{kind: string, gameId: string, target: string}} scope
+ * @param {{idempotencyKey: string, branch: string, from: string|null, head: string}} record
+ * @returns {Promise<void>}
+ */
+export async function save(repoRoot, scope, record) {
+  await mkdir(directory(repoRoot), { recursive: true });
+  await writeFile(fileFor(repoRoot, scope), JSON.stringify(record), { mode: 0o600 });
+}
+
+/**
+ * @param {string} repoRoot
+ * @param {{kind: string, gameId: string, target: string}} scope
+ * @returns {Promise<void>}
+ */
+export async function clear(repoRoot, scope) {
+  await rm(fileFor(repoRoot, scope), { force: true });
+}
+
+/**
+ * Get onto the kept branch for a rerun: it may have been switched away from or deleted.
+ * @param {{git: Object, branch: string}} options
+ * @returns {Promise<{ok: true}|{ok: false, reason: string}>}
+ */
+export async function returnToBranch({ git, branch }) {
+  if ((await git.currentBranch()) === branch) return { ok: true };
+  const back = await git.git(['switch', ...(await git.branchExists(branch) ? [] : ['-c']), branch]);
+  return back.ok ? { ok: true } : { ok: false, reason: `The branch ${branch} from the earlier attempt could not be checked out. Nothing more was charged.` };
+}

package/src/run/personaLoop.js

@@ -217,6 +217,7 @@ export async function runPersona(args) {
     // game", and a launch that hangs or dies should leave the server holding
     // that rather than still holding `queued`.
     await advance('launching');
+    if ('traceSecrets' in driver) driver.traceSecrets = secretsOf(localBrief);
     await driver.launch({ target, runDir: runDirectory, recordVideo, viewport, signal: args.signal });
     await advance('playing');
 
@@ -239,6 +240,7 @@ export async function runPersona(args) {
 
     const endedAt = new Date().toISOString();
     await driverStop(driver, outcome, log, play.canceled ? 'canceled' : 'completed');
+    if (typeof driver.playMetrics === 'function') outcome.playMetrics = driver.playMetrics();
 
     if (args.signal?.aborted || heartbeat?.canceled) play.canceled = true;
     if (play.canceled) {
@@ -413,7 +415,7 @@ async function playLoop({
   // blocker.
   let awaitingAfter = [];
   const observed = observation => {
-    const entry = { index: observations.length + 1, step: observation.step, url: observation.url || null, title: observationTitle(observation) };
+    const entry = observationEntry(observation, observations.length + 1);
     observations.push(entry);
     for (const note of awaitingAfter) note.after = { url: entry.url, title: entry.title };
     awaitingAfter = [];
@@ -543,6 +545,14 @@ async function playLoop({
     for (const toolUse of answer.toolUses) {
       if (argsSignal?.aborted || heartbeat?.canceled) { canceled = true; quitReason = 'error'; quitDetail = 'the job was cancelled'; break; }
       if (selection && toolUse.id !== selection.toolId) { results.push({ type: 'tool_result', tool_use_id: toolUse.id, content: 'Alternative not selected. No action executed.' }); continue; }
+      // One turn can send many actions, and the budget is a promise to the
+      // developer about all of them, so it is checked per action. Every tool_use
+      // still needs its tool_result, or the next message would be refused.
+      if (ACTION_TOOLS.includes(toolUse.name) && actions >= maxActions) {
+        results.push({ type: 'tool_result', tool_use_id: toolUse.id, content: `Not played: the action budget of ${maxActions} has run out.` });
+        transcript.write({ kind: 'action_not_played', tool: toolUse.name, reason: 'action_budget' });
+        continue;
+      }
       const notesBefore = notes.length;
       const executed = await executeTool({
         driver,
@@ -1012,6 +1022,15 @@ async function reportPhase({
   markdown = stampGeneratedAt(markdown, generatedAt, facts);
   // The same slots in the prose the dashboard shows from report.json.
   report = fillReportPlaceholders(report, { generatedAt, ...facts });
+  // A finding the server would drop for want of an uploaded file it can
+  // open gets the screenshot taken where it happened.
+  const withShots = attachScreenshotEvidence(report, { screenshots, notes });
+  if (withShots.attached.length > 0) {
+    report = withShots.report;
+    transcript.write({ kind: 'evidence_attached', findings: withShots.attached });
+    await writeFile(path.join(runDirectory, RUN_FILES.runnerEvidence), `${JSON.stringify({ attached: withShots.attached }, null, 2)}\n`, 'utf8');
+    log(`attached a screenshot as evidence to ${withShots.attached.length} finding(s) that cited none: ${withShots.attached.map(entry => `${entry.id} (${entry.ref})`).join(', ')}`);
+  }
   // Whatever the model wrote, the password does not leave in a report.
   markdown = scrubSecrets(markdown, secrets);
   report = scrubSecrets(report, secrets);
@@ -1466,9 +1485,16 @@ export function redactToolInput(toolUse, secrets = []) {
   return scrubSecrets(rest, secrets);
 }
 
-async function safeScreenshot(driver, name, screenshots, transcript, observation = null) {
+export async function safeScreenshot(driver, name, screenshots, transcript, observation = null) {
   try {
     const shot = await driver.screenshot(name);
+    // A driver that cannot screenshot here (headless Godot answers
+    // `{ path: null, skipped }`) saved nothing. Listing it anyway is how a
+    // report came to cite "screenshots/undefined" and "- undefined".
+    if (!shot || !shot.relativePath) {
+      transcript.write({ kind: 'screenshot_skipped', name, reason: (shot && shot.skipped) || 'no file' });
+      return null;
+    }
     screenshots.push({ ...shot, reason: name, ...(Number.isInteger(observation) ? { observation } : {}), order: nextOrder() });
     transcript.write({ kind: 'screenshot', path: shot.relativePath, reason: name });
     return shot;
@@ -1769,14 +1795,95 @@ export function screenshotLine(shot) {
 /** The most observations listed in the report prompt; the newest are kept. */
 export const MAX_OBSERVATIONS_IN_REPORT = 200;
 
-function observationLines(observations) {
+/** The most characters of one observation's game state kept for the report. */
+export const MAX_STATE_IN_REPORT_CHARS = 600;
+
+/** The most characters of game state in the whole report prompt; newest first. */
+export const REPORT_STATE_BUDGET_CHARS = 24000;
+
+/**
+ * What the report step keeps of one observation: its index, step, URL,
+ * title and, when the game exposes `_ravensight_state()`, that state. On
+ * godot_driver the title is always the scene path, so without the state the
+ * report writer saw sixty identical lines and concluded the game showed no
+ * ember, room or deaths.
+ *
+ * @param {Object} observation
+ * @param {number} index
+ * @returns {Object}
+ */
+export function observationEntry(observation, index) {
+  const entry = { index, step: observation.step, url: observation.url || null, title: observationTitle(observation) };
+  const state = typeof observation.gameStateText === 'string' ? observation.gameStateText.split('\n').map(line => line.trim()).filter(Boolean).join(' ; ') : '';
+  if (state) entry.state = state.length > MAX_STATE_IN_REPORT_CHARS ? `${state.slice(0, MAX_STATE_IN_REPORT_CHARS)}... [cut]` : state;
+  return entry;
+}
+
+export function observationLines(observations) {
   if (!observations || observations.length === 0) return '(none)';
   const shown = observations.slice(-MAX_OBSERVATIONS_IN_REPORT);
-  const lines = shown.map(entry => `observation ${entry.index}: step ${entry.step}${entry.url ? ` ${entry.url}` : ''}${entry.title ? ` "${entry.title}"` : ''}${entry.driver_error ? ` [driver error: ${entry.driver_error}]` : ''}`);
+  // Game state for the newest observations first, until the budget is spent.
+  let budget = REPORT_STATE_BUDGET_CHARS;
+  let withState = 0;
+  const states = new Map();
+  for (let i = shown.length - 1; i >= 0; i -= 1) {
+    const state = shown[i].state;
+    if (!state) continue;
+    if (state.length > budget) break;
+    budget -= state.length;
+    states.set(shown[i], state);
+    withState += 1;
+  }
+  const lines = shown.map(entry => {
+    const head = `observation ${entry.index}: step ${entry.step}${entry.url ? ` ${entry.url}` : ''}${entry.title ? ` "${entry.title}"` : ''}${entry.driver_error ? ` [driver error: ${entry.driver_error}]` : ''}`;
+    return states.has(entry) ? `${head}\n    game state: ${states.get(entry)}` : head;
+  });
+  const statefulCount = shown.filter(entry => entry.state).length;
+  if (withState < statefulCount) lines.unshift(`(game state is shown for the newest ${withState} of ${statefulCount} observations that have one)`);
   if (shown.length < observations.length) lines.unshift(`(the first ${observations.length - shown.length} are left out)`);
   return lines.join('\n');
 }
 
+/**
+ * Give every finding that cites no screenshot this run has the screenshot
+ * nearest after the latest observation it cites (a `note N` counts as that
+ * note's observation), else the last screenshot. The server keeps a finding
+ * only when one of its evidence refs names a file the run uploaded, and the
+ * transcript is uploaded only on `--upload-transcript`, so a finding that
+ * cited only `observation N` was dropped however real it was. Screenshots are
+ * uploaded by default, and this one was taken at that point of the run.
+ *
+ * @param {Object} report
+ * @param {{screenshots: Array<Object>, notes?: Array<Object>}} context
+ * @returns {{report: Object, attached: Array<{id: string, ref: string}>}}
+ */
+export function attachScreenshotEvidence(report, { screenshots = [], notes = [] } = {}) {
+  const usable = (screenshots || []).filter(shot => shot && typeof shot.relativePath === 'string' && shot.relativePath);
+  if (!report || !Array.isArray(report.findings) || usable.length === 0) return { report, attached: [] };
+  const known = new Set(usable.map(shot => shot.relativePath));
+  const attached = [];
+  const findings = report.findings.map(finding => {
+    const evidence = Array.isArray(finding && finding.evidence) ? finding.evidence : [];
+    if (evidence.some(item => item && item.type === 'screenshot' && known.has(item.ref))) return finding;
+    let latest = null;
+    for (const item of evidence) {
+      if (!item || item.type !== 'transcript_step') continue;
+      const ref = String(item.ref || '');
+      const observation = ref.match(/^observation\s+(\d+)$/i);
+      const note = ref.match(/^note\s+(\d+)$/i);
+      const at = observation ? Number(observation[1]) : note && notes[Number(note[1]) - 1] ? Number(notes[Number(note[1]) - 1].observation) : null;
+      if (Number.isFinite(at) && (latest === null || at > latest)) latest = at;
+    }
+    const after = latest === null ? [] : usable.filter(shot => Number.isFinite(shot.observation) && shot.observation >= latest);
+    const shot = after.length > 0
+      ? after.reduce((best, candidate) => (candidate.observation < best.observation || (candidate.observation === best.observation && (candidate.order ?? 0) < (best.order ?? 0)) ? candidate : best))
+      : usable[usable.length - 1];
+    attached.push({ id: finding.id, ref: shot.relativePath });
+    return { ...finding, evidence: [...evidence, { type: 'screenshot', ref: shot.relativePath }] };
+  });
+  return { report: { ...report, findings }, attached };
+}
+
 /**
  * A short title for an observation: the page title when the driver has one,
  * otherwise the first line of what was observed.