ravensight-playtest 0.4.0 → 0.5.1

package/src/run/personaLoop.js

@@ -7,7 +7,7 @@ import { withRollingCacheBreakpoint } from './fixLoop.js';
 import { callStepWithRetry, ProxyError, STEPS } from './model.js';
 import { RUN_FILES } from './paths.js';
 import { appendNormalisedMarkdown, checkEvidence, createReportValidator, lintMarkdownSections, normaliseEnums, normaliseEvidence, repairInstruction, REQUIRED_MARKDOWN_SECTIONS, stampReport, writeReportFiles } from './report.js';
-import { assertNoSecrets } from './secretScan.js';
+import { assertNoSecrets, scanForSecrets } from './secretScan.js';
 import { ACTION_TOOLS, PLAY_TOOLS, SWIPE_TOOL, REPORT_TOOLS, executeTool } from './tools.js';
 import { createTranscript } from './transcript.js';
 import { createUsageLedger, writeUsageFile } from './usage.js';
@@ -35,14 +35,17 @@ import { MIN_PASSWORD_CHARS } from '../briefLocal.js';
  * - `X-Playtest-Budget: warn` at 80 percent of the run budget starts a wrap
  *   up, so the report (which is the product) gets written while there is
  *   still budget to write it with.
- * - A 402 means the money is gone. Nothing further is asked of the model,
- *   including the report: a partial `report.md` and `report.json` are
- *   written locally from the notes (`partial: true`, each note an
+ * - An act 402 stops gameplay. If its authoritative figures show room, the
+ *   cheaper report phase may still fit. Otherwise partial `report.md` and
+ *   `report.json` files are written locally from the notes (`partial: true`, each note an
  *   unreviewed finding), `complete` is sent with `state: budget_exceeded`
  *   and that report, and the process exits 2. The notes are the one thing
  *   the run bought before the money ran out, so they are not thrown away.
  */
 
+/** Why a report is partial when the money ran out. */
+const BUDGET_PARTIAL_REASON = 'The run reached its model budget before the report could be completed. What the persona saw is below, and report.json carries the notes as unreviewed findings.';
+
 /**
  * Order across notes and screenshots, so a partial report can attach the
  * screenshot nearest to each note. Module wide and only ever compared within
@@ -161,6 +164,9 @@ export async function runPersona(args) {
   // what the screen looked like next and a finding can cite where a failure
   // is visible.
   const observations = [];
+  // What the persona did, kept out here so a report can still say so when the
+  // play loop dies part way through.
+  const actionRecords = [];
   let imageBudget = maxImages;
 
   await mkdir(path.join(runDirectory, 'screenshots'), { recursive: true });
@@ -211,7 +217,7 @@ export async function runPersona(args) {
     // game", and a launch that hangs or dies should leave the server holding
     // that rather than still holding `queued`.
     await advance('launching');
-    await driver.launch({ target, runDir: runDirectory, recordVideo, viewport });
+    await driver.launch({ target, runDir: runDirectory, recordVideo, viewport, signal: args.signal });
     await advance('playing');
 
     const play = await playLoop({
@@ -221,6 +227,7 @@ export async function runPersona(args) {
       notes,
       screenshots,
       observations,
+      actionRecords,
       imageBudget,
       keepObservations,
       onImageBudget: next => {
@@ -231,8 +238,9 @@ export async function runPersona(args) {
     outcome = { ...outcome, ...play };
 
     const endedAt = new Date().toISOString();
-    await driverStop(driver, outcome, log);
+    await driverStop(driver, outcome, log, play.canceled ? 'canceled' : 'completed');
 
+    if (args.signal?.aborted || heartbeat?.canceled) play.canceled = true;
     if (play.canceled) {
       outcome.state = 'canceled';
       outcome.quitReason = play.quitReason || 'error';
@@ -240,24 +248,17 @@ export async function runPersona(args) {
       return finish(outcome, ledger, runDirectory, transcript, heartbeat);
     }
 
-    if (play.budgetExceeded) {
+    if (play.budgetExceeded && !play.reportBudgetAvailable) {
       outcome.state = 'budget_exceeded';
       outcome.quitReason = 'cost_cap';
-      // Nothing more is asked of the model: there is no budget left to ask
-      // with, and a refused call would just be another 402.
-      // A report that cannot be written (a secret shaped note, a full disk)
-      // must not turn an out of budget run into a failed one.
-      try {
-        await writePartialReport({ outcome, runDirectory, persona, driverName, target, notes, screenshots, play, skill, ledger, runId, jobId, startedAt, endedAt });
-      } catch (writeError) {
-        outcome.report = null;
-        outcome.reportMarkdown = null;
-        log(`the partial report could not be written: ${String(writeError && writeError.message)}`);
-      }
+      // No positive remaining budget was established by the refusal.
+      // Preserve the notes without authorizing another model call.
+      await writeSomeReport({ outcome, runDirectory, persona, driverName, target, notes, screenshots, actionRecords, play, skill, ledger, runId, jobId, startedAt, endedAt, log });
       return finish(outcome, ledger, runDirectory, transcript, heartbeat);
     }
 
     await advance('reporting');
+    if (args.signal?.aborted || heartbeat?.canceled) throw new Error('The playthrough was interrupted before reporting.');
     onProgress({ phase: 'reporting', persona: persona.slug });
 
     const written = await reportPhase({
@@ -288,31 +289,48 @@ export async function runPersona(args) {
     outcome.report = written.report;
     outcome.reportMarkdown = written.markdown;
     outcome.reportLint = written.lint;
-    outcome.state = 'succeeded';
-    outcome.quitReason = written.report.quit_reason;
+    outcome.state = play.budgetExceeded ? 'budget_exceeded' : 'succeeded';
+    outcome.quitReason = play.budgetExceeded ? 'cost_cap' : written.report.quit_reason;
     outcome.quitDetail = written.report.quit_detail || '';
     return finish(outcome, ledger, runDirectory, transcript, heartbeat);
   } catch (error) {
-    await driverStop(driver, outcome, log);
+    await driverStop(driver, outcome, log, args.signal?.aborted ? 'canceled' : 'failed');
     if (error instanceof ProxyError && error.isBudget) {
       outcome.state = 'budget_exceeded';
       outcome.quitReason = 'cost_cap';
       // The money ran out in the report step itself: the notes are still
       // worth keeping, by the same rule as a 402 during play.
-      try {
-        const play = { actionsTaken: outcome.actionsTaken, quitReason: 'cost_cap', quitDetail: String(error.message || '') };
-        await writePartialReport({ outcome, runDirectory, persona, driverName, target, notes, screenshots, play, skill, ledger, runId, jobId, startedAt, endedAt: new Date().toISOString() });
-      } catch (writeError) {
-        log(`the partial report could not be written: ${String(writeError && writeError.message)}`);
-      }
+      const play = { actionsTaken: outcome.actionsTaken, quitReason: 'cost_cap', quitDetail: String(error.message || '') };
+      await writeSomeReport({ outcome, runDirectory, persona, driverName, target, notes, screenshots, actionRecords, play, skill, ledger, runId, jobId, startedAt, endedAt: new Date().toISOString(), log, preservedMarkdown: error.completedReportMarkdown });
     } else if (error instanceof ProxyError && error.isClosed) {
       outcome.state = 'canceled';
       outcome.quitReason = 'error';
     } else {
       outcome.state = 'failed';
       outcome.quitReason = 'error';
+      // A run that broke still reports what it saw and why it stopped. The
+      // state is still `failed`, so the unit is still refunded; the report
+      // costs no model call. An abort is the person stopping it, not a failure.
+      if (!args.signal?.aborted) {
+        const detail = safeErrorDetail(error, localBrief);
+        // The play loop's count only reaches `outcome` when it returns, and this
+        // may be the play loop dying.
+        const actionsTaken = Math.max(outcome.actionsTaken || 0, actionLogFrom(actionRecords).length);
+        const play = { actionsTaken, quitReason: 'error', quitDetail: detail };
+        await writeSomeReport({
+          outcome, runDirectory, persona, driverName, target, notes, screenshots, actionRecords, play, skill, ledger, runId, jobId, startedAt, endedAt: new Date().toISOString(), log,
+          quitReason: 'error',
+          // The notes stay in report.md and out of report.json. Nothing reviewed
+          // them, they may be symptoms of the crash, and what report.json holds
+          // becomes findings, analyst memory and graph nodes on the server.
+          findings: [],
+          reason: `The run stopped on an error before its report was written: ${detail}`,
+          bareReason: 'The run stopped on an error before its report was written.'
+        });
+      }
     }
-    outcome.quitDetail = String(error && error.message ? error.message : error);
+    // Sent to the server as the run's reason, so it is as safe as the report.
+    outcome.quitDetail = safeErrorDetail(error, localBrief);
     transcript.write({ kind: 'error', message: outcome.quitDetail });
     await finish(outcome, ledger, runDirectory, transcript, heartbeat);
     throw Object.assign(error, { outcome });
@@ -325,8 +343,12 @@ export async function runPersona(args) {
  * @returns {Promise<Object>}
  */
 async function playLoop({
+  // The observation and action lines of the transcript, kept for the report
+  // step's "Actions you took" list and for a partial report.
+  actionRecords = [],
   api,
   controller = 'standard',
+  signal: argsSignal,
   client,
   jobId,
   runId,
@@ -355,11 +377,11 @@ async function playLoop({
   const identity = runIdentity({ persona, runId });
   const secrets = secretsOf(localBrief);
   const noisePatterns = (localBrief && Array.isArray(localBrief.expected_noise)) ? localBrief.expected_noise : [];
-  if (controller === 'jev') system.push({ type: 'text', text: JEV_CONTROLLER_PROMPT });
+  if (['jev', 'rune'].includes(controller)) system.push({ type: 'text', text: JEV_CONTROLLER_PROMPT });
   const availableTools = driver.supportedActions
     ? [...PLAY_TOOLS, ...(driver.supportedActions.includes('swipe') ? [SWIPE_TOOL] : [])].filter(tool => !ACTION_TOOLS.includes(tool.name) || driver.supportedActions.includes(tool.name))
     : PLAY_TOOLS;
-  const playTools = controller === 'jev' ? availableTools.map(tool => ({
+  const playTools = ['jev', 'rune'].includes(controller) ? availableTools.map(tool => ({
     ...tool,
     input_schema: {
       ...tool.input_schema,
@@ -372,9 +394,6 @@ async function playLoop({
     }
   })) : availableTools;
   const recentActions = [];
-  // The observation and action lines of the transcript, kept for the report
-  // step's "Actions you took" list.
-  const actionRecords = [];
   let latestObservation = '';
   let visualObservation = false;
   const messages = [];
@@ -384,6 +403,7 @@ async function playLoop({
   let quitDetail = '';
   let budgetWarned = false;
   let budgetExceeded = false;
+  let reportBudgetAvailable = false;
   let canceled = false;
   let unchanged = 0;
   const maxTurns = maxActions * 3 + 20;
@@ -426,7 +446,7 @@ async function playLoop({
   });
 
   while (turns < maxTurns && actions < maxActions) {
-    if (heartbeat && heartbeat.canceled) {
+    if (argsSignal?.aborted || (heartbeat && heartbeat.canceled)) {
       canceled = true;
       quitReason = 'error';
       quitDetail = 'the job was cancelled';
@@ -455,6 +475,7 @@ async function playLoop({
     } catch (error) {
       if (error instanceof ProxyError && error.isBudget) {
         budgetExceeded = true;
+        reportBudgetAvailable = hasPositiveRemainingBudget(error);
         quitReason = 'cost_cap';
         quitDetail = error.message;
         break;
@@ -485,6 +506,7 @@ async function playLoop({
       if (typeof onBudgetWarning === 'function') await onBudgetWarning();
     }
 
+    if (argsSignal?.aborted || heartbeat?.canceled) { canceled = true; quitReason = 'error'; quitDetail = 'the job was cancelled'; break; }
     if (answer.toolUses.length === 0) {
       // No tool call is how a persona stops: there is no quit tool in the
       // step's allowlist, and adding one would be refused by the proxy.
@@ -495,14 +517,15 @@ async function playLoop({
     }
 
     let selection = null;
-    if (controller === 'jev') {
+    if (['jev', 'rune'].includes(controller)) {
       try {
-        selection = await selectAction({ api, jobId, runId, persona, observation: `${visualObservation ? '' : latestObservation}\nScreen interpretation: ${answer.toolUses.map(tool => tool.input?.visible_state || '').filter(Boolean).join('\n') || answer.text}`, toolUses: answer.toolUses, recentActions, intentNotes });
+        selection = await selectAction({ api, jobId, runId, persona, controller, observation: `${visualObservation ? '' : latestObservation}\nScreen interpretation: ${answer.toolUses.map(tool => tool.input?.visible_state || '').filter(Boolean).join('\n') || answer.text}`, toolUses: answer.toolUses, recentActions, intentNotes });
       } catch (error) {
         if (error instanceof ProxyError && error.isBudget) { budgetExceeded = true; quitReason = 'cost_cap'; quitDetail = error.message; break; }
         if (error instanceof ProxyError && error.isClosed) { canceled = true; quitDetail = error.message; break; }
         throw error;
       }
+      if (argsSignal?.aborted || heartbeat?.canceled) { canceled = true; quitReason = 'error'; quitDetail = 'the job was cancelled'; break; }
       transcript.write({ kind: 'action_selection', source: selection.source || 'jev', status: selection.status, confidence: selection.confidence, selected_tool_id: selection.toolId, reason: selection.reason });
       if (selection.status !== 'selected') { quitReason = 'persona_quit'; quitDetail = 'Ravensight Playtester stopped for review: ' + selection.reason; break; }
       if (selection.budget === 'warn' && !budgetWarned) {
@@ -518,6 +541,7 @@ async function playLoop({
     // its actions changes the screen, a later ref may name something else.
     let screenChangedThisTurn = false;
     for (const toolUse of answer.toolUses) {
+      if (argsSignal?.aborted || heartbeat?.canceled) { canceled = true; quitReason = 'error'; quitDetail = 'the job was cancelled'; break; }
       if (selection && toolUse.id !== selection.toolId) { results.push({ type: 'tool_result', tool_use_id: toolUse.id, content: 'Alternative not selected. No action executed.' }); continue; }
       const notesBefore = notes.length;
       const executed = await executeTool({
@@ -660,6 +684,7 @@ async function playLoop({
     quitDetail,
     budgetWarned,
     budgetExceeded,
+    reportBudgetAvailable,
     canceled,
     imageBudget,
     finalScreenshot: final ? final.relativePath : null,
@@ -773,7 +798,7 @@ async function reportPhase({
   // never escalates to a dearer tier after a warning, and a 402 on a report
   // turn gets one retry with the output ceiling cut to what the refusal says
   // still fits.
-  const escalate = shouldEscalate(persona) && !play.budgetWarned;
+  const escalate = shouldEscalate(persona) && !play.budgetWarned && !play.budgetExceeded;
   let maxTokens = REPORT_MAX_TOKENS;
   let shrunk = false;
   let lastCandidate = null;
@@ -815,6 +840,14 @@ async function reportPhase({
         log(`the model budget ran out on a follow-up report turn; keeping the report.json already written (${lint.length} problem(s)).`);
         break;
       }
+      // A completed markdown write can survive a cut-off JSON write. Keep
+      // that draft without promoting its prose into structured findings.
+      if (typeof markdown === 'string' && markdown.trim()) {
+        const safeMarkdown = scrubSecrets(markdown, secrets);
+        if (Buffer.byteLength(safeMarkdown, 'utf8') <= 256 * 1024 && scanForSecrets(safeMarkdown).length === 0) {
+          error.completedReportMarkdown = safeMarkdown;
+        }
+      }
       throw error;
     }
     ledger.record({ model: answer.model, usage: answer.usage, ms: Date.now() - askedAt });
@@ -869,7 +902,10 @@ async function reportPhase({
     if (enumFixed.changes.length > 0) {
       transcript.write({ kind: 'report_values_normalised', attempt, changes: enumFixed.changes.map(change => ({ id: change.id, field: change.field, from: change.from, to: change.to })) });
     }
-    const capped = normalised ? capNetworkOnlyFindings(stampReport(enumFixed.report, facts), { notes, screenshots }) : null;
+    const reportCandidate = play.budgetExceeded
+      ? { ...enumFixed.report, partial: true, quit_reason: 'cost_cap', quit_detail: play.quitDetail }
+      : enumFixed.report;
+    const capped = normalised ? capNetworkOnlyFindings(stampReport(reportCandidate, facts), { notes, screenshots }) : null;
     const runnerDrop = capped ? dropRunnerRejectedFindings(capped, { notes, observations }) : { report: null, dropped: [] };
     for (const entry of runnerDrop.dropped) {
       transcript.write({ kind: 'finding_dropped', id: entry.finding.id, severity: entry.finding.severity, title: entry.finding.title, reason: entry.reason });
@@ -947,7 +983,7 @@ async function reportPhase({
   // after the report is settled, so a refusal can cost the check and never
   // the report.
   report = scrubSecrets(report, secrets);
-  if (!shrunk) {
+  if (!shrunk && !play.budgetExceeded) {
     const checked = await checkClaimsAgainstScreenshots({ client, jobId, runId, report, screenshots, runDirectory, ledger, transcript, log });
     if (checked.downgraded.length > 0) {
       report = checked.report;
@@ -955,7 +991,7 @@ async function reportPhase({
       log(`downgraded ${checked.downgraded.length} finding(s) their own screenshot contradicts: ${checked.downgraded.map(entry => `${entry.id} (${entry.from} to ${entry.to}, ${entry.screenshot})`).join(', ')}`);
     }
   } else {
-    transcript.write({ kind: 'claim_check_skipped', reason: 'the report turn needed a smaller output ceiling to fit the budget' });
+    transcript.write({ kind: 'claim_check_skipped', reason: play.budgetExceeded ? 'gameplay stopped for budget; preserving the report budget' : 'the report turn needed a smaller output ceiling to fit the budget' });
   }
 
   if (!markdown) {
@@ -1116,6 +1152,13 @@ export async function checkClaimsAgainstScreenshots({ client, jobId, runId, repo
 /** The smallest output ceiling worth retrying a report turn with. */
 export const REPORT_MIN_TOKENS = 1536;
 
+/** A refused act is not proof that a smaller, cheaper report cannot fit. */
+function hasPositiveRemainingBudget(error) {
+  const { usd_used: used, usd_max: maximum, projected_usd: projected } = error.figures || {};
+  return [used, maximum, projected].every(value => typeof value === 'number' && Number.isFinite(value))
+    && used >= 0 && maximum > used && projected > maximum - used;
+}
+
 /** Whether a 402 carried the three figures a retry is sized from. */
 export function hasBudgetFigures(error) {
   const figures = (error && error.figures) || {};
@@ -1437,9 +1480,10 @@ async function safeScreenshot(driver, name, screenshots, transcript, observation
   }
 }
 
-async function driverStop(driver, outcome, log) {
+async function driverStop(driver, outcome, log, reason) {
   try {
-    const stopped = await driver.stop();
+    const stopped = await driver.stop({ reason });
+    if (stopped?.recording) outcome.recording = stopped.recording;
     if (stopped && stopped.videoPath) outcome.videoPath = stopped.videoPath;
     // The Godot driver captures frames rather than video, and stitching them
     // needs ffmpeg, which is optional. No video is a normal outcome: say
@@ -1483,7 +1527,7 @@ async function finish(outcome, ledger, runDirectory, transcript, heartbeat) {
  * screenshot list and says plainly that it is partial, rather than
  * inventing a persona verdict nobody gave.
  */
-export function renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason }) {
+export function renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, actions = [], play, reason }) {
   const good = notes.filter(note => note.kind === 'good');
   const bad = notes.filter(note => note.kind === 'bug' || note.kind === 'confusing');
   const network = notes.filter(note => note.automatic && note.url);
@@ -1502,6 +1546,7 @@ export function renderFallbackMarkdown({ persona, driverName, target, notes, scr
     '## Why this report is partial',
     reason,
     '',
+    ...(actions.length > 0 ? ['## What the persona did', actionLogLines(actions), ''] : []),
     '## What worked',
     list(good),
     '',
@@ -1575,7 +1620,7 @@ export function notesToFindings(notes, screenshots) {
       description: text,
       repro_steps: [`Seen after ${Number(note.at_action) || 0} action(s) in this run.`],
       evidence: shot
-        ? [{ type: 'screenshot', ref: shot.relativePath }]
+        ? [{ type: 'screenshot', ref: String(shot.relativePath).replace(/\\/g, '/') }]
         : [{ type: 'transcript_step', ref: `action ${Number(note.at_action) || 0}` }]
     });
   }
@@ -1589,9 +1634,9 @@ export function notesToFindings(notes, screenshots) {
  * and no persona verdict was recorded; `partial` is what tells a reader not
  * to read them as one.
  */
-async function writePartialReport({ outcome, runDirectory, persona, driverName, target, notes, screenshots, play, skill, ledger, runId, jobId, startedAt, endedAt }) {
+async function writePartialReport({ outcome, runDirectory, persona, driverName, target, notes, screenshots, actionRecords = [], play, skill, ledger, runId, jobId, startedAt, endedAt, quitReason = 'cost_cap', reason = BUDGET_PARTIAL_REASON, findings = null, preservedMarkdown = null }) {
   const report = stampReport({
-    findings: notesToFindings(notes, screenshots),
+    findings: findings || notesToFindings(notes, screenshots),
     sentiment: { fun: 3, clarity: 3, frustration: 3 },
     would_continue: false,
     partial: true
@@ -1606,24 +1651,70 @@ async function writePartialReport({ outcome, runDirectory, persona, driverName,
     actionsTaken: play.actionsTaken,
     usage: ledger.toReportUsage(),
     promptVersion: promptVersionOf(skill) || 'unknown',
-    quitReason: 'cost_cap',
+    quitReason,
     quitDetail: play.quitDetail
   });
-  const markdown = renderFallbackMarkdown({
+  const fallbackMarkdown = renderFallbackMarkdown({
     persona,
     driverName,
     target,
     notes,
     screenshots,
+    actions: actionLogFrom(actionRecords),
     play,
-    reason: 'The run reached its model budget before the report step. What the persona saw is below, and report.json carries the notes as unreviewed findings.'
+    reason
   });
+  const markdown = preservedMarkdown
+    ? ['# Playtest (partial report draft)', '',
+      'The run reached its model budget before this draft could be completed and checked. It is preserved below for review.',
+      '', preservedMarkdown, '', '---', '', fallbackMarkdown].join('\n')
+    : fallbackMarkdown;
   const stamped = stampGeneratedAt(markdown, new Date().toISOString(), reportFacts({ persona, driverName, target, play }));
   await writeReportFiles({ runDirectory, markdown: stamped, report });
   outcome.report = report;
   outcome.reportMarkdown = stamped;
 }
 
+/**
+ * An error message fit to leave this machine: the brief's own secrets replaced,
+ * and the whole message withheld if anything else in it is shaped like a secret.
+ */
+function safeErrorDetail(error, localBrief) {
+  const text = scrubSecrets(String(error && error.message ? error.message : error), secretsOf(localBrief));
+  if (scanForSecrets(text).length > 0) return 'The error message was withheld because it contained something shaped like a secret.';
+  return text.slice(0, 400);
+}
+
+/**
+ * A run always ends with a report. If the notes cannot be written (a secret
+ * shaped note, a secret in an error message), the report goes out without them
+ * and says so: the reason and the action count are worth more than nothing.
+ * Only a failure of the disk itself leaves no report.
+ */
+async function writeSomeReport({ log, bareReason = BUDGET_PARTIAL_REASON, ...report }) {
+  try {
+    await writePartialReport(report);
+    return;
+  } catch (error) {
+    log(`the partial report could not be written with its notes: ${String(error && error.message)}`);
+  }
+  try {
+    await writePartialReport({
+      ...report,
+      preservedMarkdown: null,
+      notes: [],
+      screenshots: [],
+      actionRecords: [],
+      play: { ...report.play, quitDetail: '' },
+      reason: `${bareReason} The notes were left out of this report because they could not be written safely.`
+    });
+  } catch (error) {
+    report.outcome.report = null;
+    report.outcome.reportMarkdown = null;
+    log(`the partial report could not be written: ${String(error && error.message)}`);
+  }
+}
+
 async function writePartialMarkdown({ runDirectory, persona, driverName, target, notes, screenshots, play, reason }) {
   const markdown = stampGeneratedAt(renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason }), new Date().toISOString(), reportFacts({ persona, driverName, target, play }));
   assertNoSecrets('report.md', markdown);

package/src/run/resume.js

@@ -1,6 +1,7 @@
 import { optionsFrom } from './args.js';
 import { getDeps } from './deps.js';
 import EXIT from './exit.js';
+import { runsMatchPlan } from './controllers.js';
 import { runDir as runDirFor } from './paths.js';
 import { buildPartialReport, readTranscript, shouldFinalize } from './regenerate.js';
 import { reconcileDroppedFindings } from './dropped.js';
@@ -57,6 +58,13 @@ export async function resumeJob({ api, gameId, jobId, job, runs, state, repoRoot
     log(`There is no journal for ${jobId} in this repository, so there is nothing to resume from. "ravensight-playtest upload ${jobId}" can still drain any pending uploads.`);
     return EXIT.ENVIRONMENT;
   }
+  const controllers = job.controllers?.length ? job.controllers : null;
+  if (journal.job_id !== jobId || journal.game_id !== gameId ||
+      JSON.stringify(journal.options?.controllers || null) !== JSON.stringify(controllers) ||
+      !runsMatchPlan(runs || [], controllers, [...new Set((runs || []).filter(row => row.module === 'persona_playtest').map(row => row.persona))])) {
+    log('The saved job/controller identity does not match the server. Nothing was replayed or charged; reconcile the saved job first.');
+    return EXIT.ENVIRONMENT;
+  }
   if (!journal.idempotency_key) {
     log('That journal has no idempotency key, so the per-run tokens cannot be minted again safely. Start a new job instead.');
     return EXIT.ENVIRONMENT;
@@ -142,7 +150,9 @@ export async function resumeJob({ api, gameId, jobId, job, runs, state, repoRoot
   // nothing is charged. The tokens that come back are new ones.
   const replayed = await api.jobs.register(gameId, {
     modules,
+    resume_job_id: jobId,
     personas: slugs,
+    ...(journal.options?.controllers ? { controllers: journal.options.controllers } : {}),
     driver: driverName,
     cli_version: deps.cliVersion || api.cliVersion,
     confirm_price_cents: Number.isInteger(journal.price_cents) ? journal.price_cents : 0,
@@ -154,11 +164,10 @@ export async function resumeJob({ api, gameId, jobId, job, runs, state, repoRoot
     pack_versions: { pack: pack.pack_version }
   }, { idempotencyKey: journal.idempotency_key });
 
-  if (!replayed.replayed) {
-    // A register that was NOT a replay means the key did not match a stored
-    // job, which means money just moved. Say so rather than letting it look
-    // like a free resume.
-    log('That idempotency key did not match a stored job, so a new job was registered and charged.');
+  const identity = rows => JSON.stringify(rows.map(row => [row.run_id, row.module, row.persona || null, row.controller || null]).sort((a, b) => a[0].localeCompare(b[0])));
+  if (replayed.replayed !== true || replayed.job_id !== jobId || identity(replayed.runs || []) !== identity(runs || [])) {
+    log('The server did not replay the exact saved job and controller runs. No playthrough was started. Reconcile this response before retrying.');
+    return EXIT.FAILED;
   }
 
   log(`resuming ${jobId}: ${outstanding.length} run(s) to play again of ${(runs || []).length}.`);
@@ -166,6 +175,8 @@ export async function resumeJob({ api, gameId, jobId, job, runs, state, repoRoot
   const resumeOptions = {
     ...options,
     controller: journal.options?.controller || options.controller || 'standard',
+    controllers,
+    recordVideo: options.recordVideo ?? journal.options?.recordVideo,
     maxActions: (journal.options && journal.options.maxActions) ?? options.maxActions,
     // The journal's answer, never the flag: a job is long or standard for good.
     runLength: journal.options?.runLength === 'long' ? 'long' : 'standard',

package/src/upload/index.js

@@ -1,3 +1,4 @@
+import { recordingCompletionBody } from './recording.js';
 import { readdir, readFile, stat } from 'node:fs/promises';
 import { join, sep } from 'node:path';
 import { ApiError, isRetryable } from '../api/errors.js';
@@ -459,10 +460,11 @@ export async function finalizeRun(runDir, options) {
   const { api, gameId, jobId, runId, body = {} } = options;
   if (!runId) throw new TypeError('finalizeRun needs a runId');
 
+  const completionBody = await recordingCompletionBody(runDir, body, options.includeVideo);
   await uploadRunDir(runDir, options);
 
   try {
-    return await api.runs.complete(gameId, jobId, runId, body);
+    return await api.runs.complete(gameId, jobId, runId, completionBody);
   } catch (error) {
     if (!(error instanceof ApiError) || error.status !== 422 || error.code !== 'upload_unverified') {
       throw error;
@@ -470,6 +472,6 @@ export async function finalizeRun(runDir, options) {
     const paths = unverifiedPaths(error);
     if (paths.length === 0) throw error;
     await uploadRunDir(runDir, { ...options, only: paths, force: true });
-    return api.runs.complete(gameId, jobId, runId, body);
+    return api.runs.complete(gameId, jobId, runId, completionBody);
   }
 }

package/src/upload/recording.js

@@ -0,0 +1,56 @@
+import { open } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { join } from 'node:path';
+
+// Deliberately independent of the full local driver receipt. Never send paths,
+// timestamps, targets, driver errors or recording.json as an artifact.
+const STATUSES = ['complete', 'partial', 'unsupported', 'failed'];
+const STOP_REASONS = [
+  'stopped', 'completed', 'canceled', 'failed', 'game_exited', 'game_crashed',
+  'page_closed', 'duration_limit', 'size_limit', 'frame_size_limit',
+  'frame_bytes_limit', 'capture_failed', 'launch_failed', 'headless_no_viewport',
+  'run_directory_required', 'encoder_unavailable', 'no_frames',
+  'encoding_timeout', 'encoding_failed', 'video_finalization_failed',
+  'video_unavailable', 'unknown'
+];
+const MAX_METADATA_BYTES = 8192;
+const MAX_DURATION_MS = 900000;
+
+export function recordingForCompletion(metadata) {
+  if (!metadata || typeof metadata !== 'object' || !STATUSES.includes(metadata.status)) return null;
+  return {
+    status: metadata.status,
+    duration_ms: typeof metadata.durationMs === 'number' && Number.isFinite(metadata.durationMs)
+      && metadata.durationMs >= 0 && metadata.durationMs <= MAX_DURATION_MS ? metadata.durationMs : null,
+    stop_reason: STOP_REASONS.includes(metadata.reason) ? metadata.reason : 'unknown'
+  };
+}
+
+async function readRecording(runDir) {
+  let file;
+  try {
+    file = await open(join(runDir, 'recording.json'), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+    const info = await file.stat();
+    if (!info.isFile() || info.size > MAX_METADATA_BYTES) return null;
+    // One bounded read also protects against a file growing after stat().
+    const buffer = Buffer.alloc(MAX_METADATA_BYTES + 1);
+    const { bytesRead } = await file.read(buffer, 0, buffer.length, 0);
+    if (bytesRead > MAX_METADATA_BYTES) return null;
+    return recordingForCompletion(JSON.parse(buffer.subarray(0, bytesRead).toString('utf8')));
+  } catch {
+    // Missing or malformed local status must not lose the run's existing report.
+    return null;
+  } finally {
+    await file?.close().catch(() => {});
+  }
+}
+
+export async function recordingCompletionBody(runDir, body, includeVideo) {
+  const { recording: supplied, include_video: ignored, ...result } = body;
+  if (includeVideo !== true) return result;
+  const recording = supplied ? recordingForCompletion({
+    status: supplied.status, durationMs: supplied.duration_ms, reason: supplied.stop_reason
+  }) : await readRecording(runDir);
+  if (recording) Object.assign(result, { include_video: true, recording });
+  return result;
+}