ravensight-playtest 0.3.2 → 0.5.0

package/src/commands/run.js

@@ -1,8 +1,10 @@
+import { recordingForCompletion, recordingCompletionBody } from '../upload/recording.js';
 import { randomUUID } from 'node:crypto';
 import { mkdir, readFile, stat } from 'node:fs/promises';
 import path from 'node:path';
 import { buildAggregate, writeAggregate } from '../run/aggregate.js';
 import { optionsFrom, unknownOptionNotes } from '../run/args.js';
+import { controllerPlan, quoteMatchesPlan, runsMatchPlan } from '../run/controllers.js';
 import { getDeps } from '../run/deps.js';
 import { createDriver } from '../run/drivers/index.js';
 import EXIT from '../run/exit.js';
@@ -94,10 +96,13 @@ async function run(flags, injected, deps, log) {
   if (setup.code !== undefined) return setup.code;
   const { context, gameId, repoRoot, apiUrl, cliVersion, api } = setup;
 
-  if (options.controller && !['standard', 'jev'].includes(options.controller)) { log('Choose --controller standard or --controller jev.'); return EXIT.ENVIRONMENT; }
+  let controllers;
+  try { controllers = controllerPlan(options); } catch (error) { log(error.message); return EXIT.ENVIRONMENT; }
+  options.controllers = controllers;
   const keyModifier = resolveKeyModifier(options.keyModifier);
   if (keyModifier.error) { log(keyModifier.error); return EXIT.ENVIRONMENT; }
   const driverName = options.driver || (context.config && context.config.driver) || 'playwright_web';
+  if (controllers?.length > 1 && !['playwright_web', 'godot_driver'].includes(driverName)) { log('Comparisons currently require a web or Godot build. Nothing was charged.'); return EXIT.ENVIRONMENT; }
   const target = resolveTarget(options, context.config, driverName);
   if (!target) {
     log('Point the run at a build: --build-url for a web build, --godot-project for Godot, or --driver-config for a native or text bridge.');
@@ -147,7 +152,7 @@ async function run(flags, injected, deps, log) {
   }
   const modules = [...RUN_MODULES];
   const pack = await deps.packs.get({ api, gameId, modules });
-  if (options.controller === 'jev' && !pack.routing?.action_controllers?.includes('jev')) { log('This server does not currently support the requested action controller. No job was registered.'); return EXIT.ENVIRONMENT; }
+  if (controllers && (!pack.routing?.controller_plans || controllers.some(controller => !pack.routing?.action_controllers?.includes(controller)))) { log('This server does not currently support the requested controller plan. No job was registered or charged.'); return EXIT.ENVIRONMENT; }
   const personaSlugs = resolvePersonas(options, context.config, pack);
   const unknownSlugs = unknownPersonas(personaSlugs, pack.personas || []);
   if (unknownSlugs.length > 0) {
@@ -161,7 +166,7 @@ async function run(flags, injected, deps, log) {
   // fetch, nothing charged. `pack.routing` is the clientRouting subset, and an
   // older cached pack without `max_personas` skips the check rather than
   // refusing on a field that does not exist yet.
-  const personaRefusal = personaCountRefusal(personaSlugs.length, pack.routing && pack.routing.max_personas);
+  const personaRefusal = personaCountRefusal(personaSlugs.length * (controllers?.length || 1), pack.routing && pack.routing.max_personas);
   if (personaRefusal) {
     log(personaRefusal);
     return EXIT.ENVIRONMENT;
@@ -192,7 +197,7 @@ async function run(flags, injected, deps, log) {
   const runLength = options.runLength === 'long' ? 'long' : 'standard';
   let estimate;
   try {
-    estimate = await api.jobs.estimate(gameId, { modules, personas: personaSlugs, runLength });
+    estimate = await api.jobs.estimate(gameId, { modules, personas: personaSlugs, runLength, ...(controllers ? { controllers, driver: driverName } : {}) });
   } catch (error) {
     if (error && error.code === 'invalid_estimate') {
       log(`The server refused to price this job: ${String(error.message || 'invalid estimate')}${runLength === 'long' ? ' This server may not price long runs yet; run without --long.' : ''} Nothing was registered or charged.`);
@@ -200,6 +205,9 @@ async function run(flags, injected, deps, log) {
     }
     throw error;
   }
+  if (!quoteMatchesPlan(estimate, controllers, personaSlugs)) { log('The server did not quote every requested controller run. Nothing was registered or charged.'); return EXIT.ENVIRONMENT; }
+  if (controllers) log(`Controllers: ${controllers.join(' + ')}. ${personas.length * controllers.length} independent playthrough(s); each run is included in the total quote.`);
+  if (controllers?.length > 1) log('Each controller gets a fresh game process or browser session. Server-side game progress and randomness may differ; comparison is evidence, not proof of identical starting state.');
   // A long run is its own unit on the server. An older server that ignores
   // run_length prices a standard run, and confirming that would buy a run
   // the CLI then plays as long.
@@ -245,6 +253,7 @@ async function run(flags, injected, deps, log) {
     registered = await api.jobs.register(gameId, {
       modules,
       personas: personaSlugs,
+      ...(controllers ? { controllers } : {}),
       driver: driverName,
       cli_version: cliVersion,
       confirm_price_cents: estimate.estimate_cents,
@@ -280,6 +289,8 @@ async function run(flags, injected, deps, log) {
       runLength,
       ...(runLength === 'long' ? { longRunFactor: options.longRunFactor } : {}),
       controller: options.controller || 'standard',
+      ...(controllers ? { controllers } : {}),
+      recordVideo: Boolean(options.recordVideo || options.uploadVideo),
       concurrency: options.concurrency ?? DEFAULT_CONCURRENCY,
       uploadVideo: Boolean(options.uploadVideo),
       uploadTranscript: Boolean(options.uploadTranscript)
@@ -291,6 +302,10 @@ async function run(flags, injected, deps, log) {
     }
   }
   await stateFile.save(repoRoot, journal);
+  if (!runsMatchPlan(registered.runs || [], controllers, personaSlugs)) {
+    log(`The registered controller plan does not match the quote. Job ${jobId} is saved for reconciliation; no playthrough or model call was started.`);
+    return EXIT.FAILED;
+  }
   log(`job ${jobId}${registered.replayed ? ' (replayed)' : ''}: ${formatCents(registered.price && registered.price.price_cents)} charged. ${directory}`);
 
   return playJob({
@@ -451,7 +466,7 @@ export async function playJob({
 
   const results = [];
   const queue = personaRuns.filter(entry => !stateFile.isRunDone(journal.runs[entry.run_id]));
-  const concurrency = WEBDRIVER_NAMES.includes(driverName) ? 1 : Math.max(1, Math.min(Number(options.concurrency) || DEFAULT_CONCURRENCY, queue.length || 1));
+  const concurrency = (options.controllers?.length > 1 || WEBDRIVER_NAMES.includes(driverName)) ? 1 : Math.max(1, Math.min(Number(options.concurrency) || DEFAULT_CONCURRENCY, queue.length || 1));
 
   /**
    * Runs the process is in the middle of. A Ctrl-C has to move these to
@@ -462,7 +477,8 @@ export async function playJob({
    * back out of, which is exactly what it is for.
    */
   const inFlight = new Map();
-  const interrupt = createInterruptHandler({ api, gameId, jobId, repoRoot, journal, inFlight, log, injected });
+  const abort = new AbortController();
+  const interrupt = createInterruptHandler({ api, gameId, jobId, repoRoot, journal, inFlight, log, injected, onInterrupt: () => abort.abort() });
 
   try {
     await Promise.all(Array.from({ length: concurrency }, async () => {
@@ -483,7 +499,7 @@ export async function playJob({
           pack,
           driverName,
           target,
-          options,
+          options: { ...options, signal: abort.signal },
           journal,
           inFlight,
           makeDriver,
@@ -633,7 +649,8 @@ export async function playPersonaRun({
   try {
     if (inFlight) inFlight.set(runId, { state: 'launching' });
     outcome = await runPersona({
-      controller: options.controller || 'standard',
+      controller: entry.controller || options.controller || 'standard',
+      signal: options.signal,
       client,
       api,
       gameId,
@@ -654,7 +671,7 @@ export async function playPersonaRun({
       maxActions,
       heartbeat,
       ledger,
-      recordVideo: Boolean(options.uploadVideo),
+      recordVideo: Boolean(options.recordVideo || options.uploadVideo),
       // Written the moment the warning arrives, not after the run: a run that
       // then fails still says in state.json that it was warned.
       onBudgetWarning: async () => {
@@ -668,6 +685,10 @@ export async function playPersonaRun({
     });
   } catch (error) {
     heartbeat.stop();
+    if (options.signal?.aborted) {
+      await stateFile.patchRun(repoRoot, journal, runId, { state: 'interrupted', completed: false, actions_taken: error.outcome?.actionsTaken || 0 });
+      return { runId, persona: persona.slug, state: 'interrupted', error };
+    }
     if (inFlight) inFlight.delete(runId);
     const failed = error.outcome || { state: 'failed', quitReason: 'error', actionsTaken: 0, report: null, reportLint: [], reachedState: 'playing' };
     await stateFile.patchRun(repoRoot, journal, runId, {
@@ -726,6 +747,10 @@ export async function playPersonaRun({
     return { runId, persona: persona.slug, state: failed.state, quitReason: failed.quitReason, error };
   }
   heartbeat.stop();
+  if (options.signal?.aborted) {
+    await stateFile.patchRun(repoRoot, journal, runId, { state: 'interrupted', completed: false, actions_taken: outcome.actionsTaken, report_written: Boolean(outcome.report) });
+    return { runId, persona: persona.slug, state: 'interrupted' };
+  }
   if (inFlight) inFlight.delete(runId);
 
   await stateFile.patchRun(repoRoot, journal, runId, {
@@ -804,6 +829,9 @@ export async function deliver({ deps, api, gameId, jobId, runId, directory, outc
   // The run card's duration. The server lane accepts `duration`; an older
   // server ignores a key it does not know.
   if (outcome.duration) body.duration = durationBody(outcome.duration);
+  // Keep the same bounded opt-in status in the older-server partial-report fallback.
+  const recording = options.uploadVideo === true ? recordingForCompletion(outcome.recording) : null;
+  Object.assign(body, await recordingCompletionBody(directory, recording ? { recording } : {}, options.uploadVideo === true));
   if (outcome.report) body.report = outcome.report;
   // `report_md` is what the server lints for its four required sections. Not
   // sending it means `report_lint` is silently empty on every run, which reads
@@ -1135,7 +1163,7 @@ async function publishAggregate({ deps, api, gameId, jobId, repoRoot, directory,
  *
  * @returns {{signalled: boolean, settle: () => Promise<void>, release: () => void}}
  */
-export function createInterruptHandler({ api, gameId, jobId, repoRoot, journal, inFlight, log, message = null, injected = {} }) {
+export function createInterruptHandler({ api, gameId, jobId, repoRoot, journal, inFlight, log, message = null, injected = {}, onInterrupt = () => {} }) {
   const signals = injected.signals || ['SIGINT', 'SIGTERM'];
   const target = injected.process || process;
   const state = { signalled: false, signal: null };
@@ -1144,6 +1172,7 @@ export function createInterruptHandler({ api, gameId, jobId, repoRoot, journal,
     if (state.signalled) return;
     state.signalled = true;
     state.signal = signal;
+    onInterrupt(signal);
     log(`\n${signal}: ${message || `finishing the turn in flight, then stopping. Nothing is lost; "ravensight-playtest resume ${jobId}" carries on.`}`);
     // Hand the signal back to the default behaviour, so a second press kills
     // the process rather than being swallowed by a handler that is now done.
@@ -1285,15 +1314,15 @@ export async function loadRunReports(repoRoot, jobId, personaRuns, journal = nul
     // summary must not count findings the dashboard does not have.
     const recorded = journal && journal.runs ? journal.runs[entry.run_id] : null;
     if (recorded && recorded.report_refused === true) {
-      out.push({ runId: entry.run_id, persona: entry.persona, state: 'failed', report: null, reportRefused: true });
+      out.push({ runId: entry.run_id, persona: entry.persona, ...(entry.controller ? { controller: entry.controller } : {}), state: 'failed', report: null, reportRefused: true });
       continue;
     }
     const file = path.join(runDirFor(repoRoot, jobId, entry.run_id), RUN_FILES.reportJson);
     try {
       const report = JSON.parse(await readFile(file, 'utf8'));
-      out.push({ runId: entry.run_id, persona: entry.persona, state: report && report.partial === true ? 'budget_exceeded' : 'succeeded', report });
+      out.push({ runId: entry.run_id, persona: entry.persona, ...(entry.controller ? { controller: entry.controller } : {}), state: recorded?.state || entry.state || (report && report.partial === true ? 'budget_exceeded' : 'succeeded'), report });
     } catch {
-      out.push({ runId: entry.run_id, persona: entry.persona, state: 'failed', report: null });
+      out.push({ runId: entry.run_id, persona: entry.persona, ...(entry.controller ? { controller: entry.controller } : {}), state: 'failed', report: null });
     }
   }
   return out;

package/src/commands/simulate.js

@@ -0,0 +1,298 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { mkdir, open, readFile, rename, stat, unlink, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { createContext, resolveGameId, CLI_VERSION, ui } from '../api/index.js';
+import { createHeartbeat } from '../run/heartbeat.js';
+import { readBridgeConfig } from '../run/drivers/stdio.js';
+import { runBatch, replayTrace, validateConfig, probeBridge } from '../simulation/index.js';
+import { jobLink } from '../dashboard.js';
+import { CliError } from '../errors.js';
+import EXIT from '../run/exit.js';
+
+const MODULES = ['balance_sim', 'aggregate_report'];
+const TERMINAL = new Set(['succeeded', 'failed', 'canceled', 'budget_exceeded', 'skipped', 'stalled']);
+const money = cents => `$${(cents / 100).toFixed(2)}`;
+const hash = value => createHash('sha256').update(JSON.stringify(value)).digest('hex');
+async function loadBridge(file) {
+  const bridge = await readBridgeConfig(file);
+  return { ...bridge, cwd: bridge.cwd || path.dirname(file) };
+}
+
+export function publicConfig(config) {
+  return { version: config.version, seed: config.seed, runs: config.runs, max_steps: config.max_steps,
+    workers: config.workers, strategies: config.strategies.map(({ name, policy }) => ({ name, policy })) };
+}
+
+async function jsonFile(file, maxBytes = 64000) {
+  if ((await stat(file)).size > maxBytes) throw new CliError(`File exceeds ${maxBytes} bytes: ${file}`);
+  return JSON.parse(await readFile(file, 'utf8'));
+}
+
+async function save(file, value) {
+  await mkdir(path.dirname(file), { recursive: true, mode: 0o700 });
+  const temporary = `${file}.${randomUUID()}.tmp`;
+  try {
+    await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600, flag: 'wx' });
+    await rename(temporary, file);
+  } finally { await unlink(temporary).catch(() => {}); }
+}
+
+// Exclusive per-journal execution. A dead process's lock is recoverable; a live
+// process cannot have its paid run restarted underneath it by another shell.
+async function acquire(file) {
+  const lock = `${file}.lock`;
+  await mkdir(path.dirname(file), { recursive: true, mode: 0o700 });
+  for (let attempt = 0; attempt < 2; attempt++) {
+    try {
+      const handle = await open(lock, 'wx', 0o600);
+      await handle.writeFile(String(process.pid));
+      await handle.close();
+      return () => unlink(lock).catch(() => {});
+    } catch (error) {
+      if (error.code !== 'EEXIST') throw error;
+      const pid = Number(await readFile(lock, 'utf8'));
+      if (!Number.isInteger(pid) || pid < 1) throw new CliError(`Unreadable simulation lock: ${lock}`);
+      try { process.kill(pid, 0); }
+      catch (probe) {
+        if (probe.code === 'ESRCH' && attempt === 0) { await unlink(lock); continue; }
+      }
+      throw new CliError('This simulation journal is already in use. Wait for the other runner to stop.');
+    }
+  }
+}
+
+function assertQuote(quote) {
+  if (quote?.execution_mode !== 'direct_command_batch' || !quote.simulation_limits ||
+      !Number.isSafeInteger(quote.estimate_cents) || quote.estimate_cents <= 0 ||
+      quote.breakdown?.length !== 1 || quote.breakdown[0].unit !== 'playtest_run' ||
+      quote.breakdown[0].count !== 1 || quote.breakdown[0].unit_cents !== quote.estimate_cents) {
+    throw new CliError('This server did not quote a paid Simulation Runner batch. Nothing was registered or charged. Update the server and CLI together.');
+  }
+}
+
+/** One confirmed purchase, a local batch, then bounded evidence to the same job. */
+export async function simulateCommand(flags = {}, injected = {}) {
+  const output = injected.ui || ui;
+  const repoRoot = path.resolve(flags.repoRoot || process.cwd());
+  let journal, file;
+  if (flags.resume) {
+    if (flags.configFile || flags.driverConfig) throw new CliError('Resume uses the original config and bridge. Do not combine --resume with --config or --driver-config.');
+    file = path.resolve(repoRoot, flags.resume);
+    journal = await jsonFile(file, 2 * 1024 * 1024);
+    if (journal.version !== 1 || journal.kind !== 'simulation' || !journal.idempotency_key ||
+        !journal.game_id || !journal.api_url || !journal.config || !journal.registration) {
+      throw new CliError('This is not a Simulation Runner journal.');
+    }
+    if (flags.game && flags.game !== journal.game_id) throw new CliError('Resume must use the original game.');
+    if (flags.apiUrl && flags.apiUrl.replace(/\/+$/, '') !== journal.api_url) throw new CliError('Resume must use the original API host.');
+  } else {
+    if (!flags.configFile) throw new CliError('Supply --config simulation.json, or --resume <state.json>.');
+    const configFile = path.resolve(repoRoot, flags.configFile);
+    const { driver_config: configDriver, ...raw } = await jsonFile(configFile);
+    const config = validateConfig(raw);
+    const driverFile = flags.driverConfig ? path.resolve(repoRoot, flags.driverConfig)
+      : typeof configDriver === 'string' ? path.resolve(path.dirname(configFile), configDriver) : null;
+    if (!driverFile) throw new CliError('Supply --driver-config bridge.json or driver_config in the simulation config.');
+    const bridge = await loadBridge(driverFile);
+    journal = { version: 1, kind: 'simulation', idempotency_key: randomUUID(), config,
+      bridge_file: driverFile, bridge_hash: hash(bridge), created_at: new Date().toISOString(), finished: false };
+    file = path.join(repoRoot, '.ravensight', 'simulations', journal.idempotency_key, 'state.json');
+  }
+  validateConfig(journal.config);
+  const context = await (injected.createContext || createContext)({ apiUrl: journal.api_url || flags.apiUrl, repoRoot, requireAuth: true });
+  if (journal.api_url && context.apiUrl.replace(/\/+$/, '') !== journal.api_url) throw new CliError('The API host changed. Resume was refused.');
+  const api = context.api;
+  const gameId = journal.game_id || resolveGameId(flags, context.config);
+  const release = await acquire(file);
+  const controller = new AbortController();
+  const target = injected.process || process;
+  const stop = () => controller.abort(new Error('Simulation canceled.'));
+  target.once('SIGINT', stop);
+  target.once('SIGTERM', stop);
+  let heartbeat;
+  const persist = () => save(file, journal);
+  const settledExit = state => journal.canceled || state === 'canceled' ? EXIT.CANCELED
+    : state !== 'succeeded' || (journal.summary && (!journal.summary.completed_runs || journal.summary.failed_runs || journal.summary.truncated_runs)) ? EXIT.FAILED : EXIT.OK;
+  const recover = () => output.error(`Recover this same purchase with: ravensight-playtest simulate --resume ${JSON.stringify(file)}`);
+  try {
+    if (journal.finished) {
+      if (flags.json) output.json({ job_id: journal.job_id, finished: true, summary: journal.summary });
+      else output.info(`This batch is already finished: ${jobLink(gameId, journal.job_id, context.apiUrl)}`);
+      return journal.exit_code ?? EXIT.OK;
+    }
+    if (journal.job_id) {
+      const { job } = await api.jobs.get(gameId, journal.job_id);
+      if (TERMINAL.has(job.state)) {
+        journal.finished = true;
+        journal.exit_code = settledExit(job.state);
+        await persist();
+        if (flags.json) output.json({ job_id: journal.job_id, finished: true, state: job.state });
+        else output.info(`The server already settled this batch as ${job.state}. No simulation was restarted.`);
+        return journal.exit_code;
+      }
+      // A completed run can await job settlement after an interrupted finish.
+      // Neither adapter files nor local execution are needed to settle it.
+      if (!journal.summary && journal.run_id) {
+        const { run: existing } = await api.runs.get(gameId, journal.job_id, journal.run_id);
+        if (TERMINAL.has(existing.state)) {
+          await api.jobs.finish(gameId, journal.job_id, {});
+          journal.finished = true; journal.exit_code = settledExit(existing.state); await persist();
+          if (flags.json) output.json({ job_id: journal.job_id, finished: true, state: existing.state });
+          else output.info(`This run already finished as ${existing.state}; its job is now settled.`);
+          return journal.exit_code;
+        }
+      }
+      if (job.cancel_requested) controller.abort(new Error('Simulation canceled on the server.'));
+    }
+    let bridge;
+    if (!journal.summary) {
+      bridge = await loadBridge(journal.bridge_file);
+      if (hash(bridge) !== journal.bridge_hash) throw new CliError('The bridge configuration changed. Restore it before resuming this purchase.');
+      if (!controller.signal.aborted) await (injected.probeBridge || probeBridge)({ bridge, seed: journal.config.seed,
+        strategy: journal.config.strategies[0].name, max_steps: journal.config.max_steps, signal: controller.signal });
+    }
+    if (!journal.registration) {
+      const simulation = publicConfig(journal.config);
+      const quote = await api.jobs.estimate(gameId, { modules: MODULES, personas: [], simulation });
+      assertQuote(quote);
+      output.info(`One simulation batch: ${journal.config.runs} games across ${journal.config.strategies.length} strategies, ${money(quote.estimate_cents)} total from prepaid balance.`);
+      output.info('Game execution and replay traces stay local. Only the bounded summary is uploaded. This is a mechanics test, not a test of the player interface.');
+      if (quote.ok === false) throw new CliError(`Insufficient prepaid balance (${money(quote.balance_cents)}). Nothing was charged.`, EXIT.ENVIRONMENT);
+      if (!flags.yes && !await (injected.confirm || output.confirm)(`Start this batch for ${money(quote.estimate_cents)}?`)) return EXIT.CANCELED;
+      if (controller.signal.aborted) return EXIT.CANCELED;
+      journal.api_url = context.apiUrl.replace(/\/+$/, '');
+      journal.game_id = gameId;
+      journal.limits = quote.simulation_limits;
+      journal.registration = { modules: MODULES, personas: [], driver: 'cli_stdio', simulation,
+        cli_version: CLI_VERSION, confirm_price_cents: quote.estimate_cents,
+        build: { kind: 'simulation', ref: hash(simulation) } };
+      // Durable BEFORE the money request. A lost register response replays the
+      // same idempotency key, even when no job_id ever reached this machine.
+      await persist();
+      output.info(`Recovery journal: ${file}`);
+    }
+
+    let registered;
+    try { registered = await api.jobs.register(gameId, journal.registration, { idempotencyKey: journal.idempotency_key }); }
+    catch (error) { recover(); throw error; }
+    journal.job_id = registered.job_id;
+    const run = registered.runs?.find(entry => entry.module === 'balance_sim');
+    if (!run || !run.playtest_token) {
+      await api.jobs.finish(gameId, journal.job_id, { state: 'failed', reason: 'Simulation run or synthetic token missing.' });
+      throw new CliError('The server did not create an executable simulation run. The job was stopped.');
+    }
+    journal.run_id = run.run_id;
+    await persist();
+    let current = flags.resume ? (await api.runs.get(gameId, journal.job_id, run.run_id)).run : run;
+    if (TERMINAL.has(current.state)) {
+      await api.jobs.finish(gameId, journal.job_id, {});
+      journal.finished = true;
+      journal.exit_code = settledExit(current.state);
+      await persist();
+      if (flags.json) output.json({ job_id: journal.job_id, finished: true, state: current.state });
+      else output.info(`This run already finished as ${current.state}; its job is now settled.`);
+      return journal.exit_code;
+    }
+    const transition = async state => {
+      if (current.state !== state) {
+        await api.runs.transition(gameId, journal.job_id, run.run_id, state);
+        current = { ...current, state };
+      }
+    };
+    heartbeat = (injected.createHeartbeat || createHeartbeat)({ api, gameId, jobId: journal.job_id, runId: run.run_id,
+      onCancel: stop, onError: error => {
+        if ([401, 403, 404, 409].includes(error.status)) controller.abort(new Error('The server closed this run.'));
+      } });
+    heartbeat.start();
+    await heartbeat.beat();
+    if (!journal.summary) {
+      if (['reporting', 'uploading'].includes(current.state)) throw new CliError('The local simulation summary is missing. Restore the journal before finishing this paid run.');
+      if (['queued', undefined].includes(current.state)) await transition('launching');
+      await transition('playing');
+      try {
+        // A crash can leave an incomplete results.jsonl. Preserve it and use a
+        // new directory for the deterministic restart of the same paid run.
+        journal.attempt_directory = path.join(path.dirname(file), `attempt-${randomUUID()}`);
+        await persist();
+        const result = await (injected.runBatch || runBatch)({ config: journal.config,
+          bridge: { ...bridge, batchTimeoutMs: Math.min(900000, (run.limits?.wall_seconds || run.budget?.wall_seconds_max || journal.limits?.wall_seconds_max || 900) * 1000) },
+          outputDir: journal.attempt_directory,
+          playtest: { token: run.playtest_token, runId: run.run_id, jobId: journal.job_id }, signal: controller.signal,
+          onProgress: progress => {
+            if (progress.finished_runs % 100 === 0 || progress.finished_runs === progress.requested_runs) {
+              output.info(`${progress.finished_runs}/${progress.requested_runs} games processed; ${progress.completed_runs} completed, ${progress.failed_runs} failed, ${progress.truncated_runs} reached a limit.`);
+              void heartbeat.beat({ actionsTaken: progress.total_steps, checkpointStep: progress.finished_runs });
+            }
+          } });
+        journal.summary = result.summary;
+        journal.traces = result.traces;
+        journal.local_report = result.report;
+        journal.canceled = controller.signal.aborted;
+        await persist();
+      } catch (error) {
+        // No usable local result exists. Complete and settle through the same
+        // server refund path; neither a second charge nor a local refund.
+        try {
+          await api.runs.complete(gameId, journal.job_id, run.run_id, { state: 'failed', quit_reason: 'error', reason: 'Local simulation could not produce a summary.' });
+          await api.jobs.finish(gameId, journal.job_id, { reason: 'Local simulation failed.' });
+          journal.finished = true; journal.exit_code = EXIT.FAILED; await persist();
+        } catch { recover(); }
+        throw error;
+      }
+    }
+    const useful = journal.summary.completed_runs > 0;
+    try {
+      if (useful) {
+        if (current.state !== 'uploading') await transition('reporting');
+        await transition('uploading');
+      }
+      await api.runs.complete(gameId, journal.job_id, run.run_id, {
+        state: useful ? 'succeeded' : 'failed', simulation: journal.summary,
+        actions_taken: journal.summary.total_steps, quit_reason: useful ? 'goal_reached' : 'error'
+      });
+      await api.jobs.finish(gameId, journal.job_id, {});
+    } catch (error) {
+      // Keep valid results for retry. A dropped complete/finish response is
+      // ambiguous, so do not overwrite success with failed or buy another job.
+      recover();
+      throw error;
+    }
+    journal.finished = true;
+    journal.exit_code = journal.canceled ? EXIT.CANCELED :
+      (!useful || journal.summary.failed_runs || journal.summary.truncated_runs) ? EXIT.FAILED : EXIT.OK;
+    await persist();
+    if (flags.json) output.json({ job_id: journal.job_id, summary: journal.summary, traces: journal.traces, journal: file });
+    else {
+      output.info(`Simulation report: ${jobLink(gameId, journal.job_id, context.apiUrl)}`);
+      output.info(`Local summary and replay traces: ${path.dirname(file)}`);
+      if (!useful) output.info('No game reached a terminal result. The server settles this batch without keeping its Playtest unit.');
+    }
+    return journal.exit_code;
+  } finally {
+    heartbeat?.stop();
+    target.removeListener('SIGINT', stop);
+    target.removeListener('SIGTERM', stop);
+    await release();
+  }
+}
+
+export async function replayCommand(trace, flags = {}, injected = {}) {
+  const output = injected.ui || ui;
+  const repo = path.resolve(flags.repoRoot || process.cwd());
+  const bridge = await loadBridge(path.resolve(repo, flags.driverConfig));
+  const controller = new AbortController();
+  const stop = () => controller.abort(new Error('Replay canceled.'));
+  const target = injected.process || process;
+  target.once('SIGINT', stop); target.once('SIGTERM', stop);
+  try {
+    const result = await (injected.replayTrace || replayTrace)({ trace: path.resolve(repo, trace), bridge, signal: controller.signal });
+    if (flags.json) output.json(result);
+    else output.info(result.matched
+      ? result.fully_replayed === false
+        ? result.failure_reproduced ? 'Replay matched the recorded prefix and reproduced the recorded failure.'
+          : 'Replay matched the recorded prefix only. The trace does not establish a complete playthrough.'
+        : 'Replay matched every recorded state, outcome and metric.'
+      : `Replay diverged: ${JSON.stringify(result.first_divergence)}. Check the game seed, build and external dependencies.`);
+    return result.matched ? EXIT.OK : EXIT.FAILED;
+  } finally { target.removeListener('SIGINT', stop); target.removeListener('SIGTERM', stop); }
+}

package/src/run/actionSelector.js

@@ -1,5 +1,5 @@
 import { ProxyError, STEPS } from './model.js';
-export const CONTROLLERS = ['standard', 'jev'];
+export { CONTROLLERS } from './controllers.js';
 export const JEV_CONTROLLER_PROMPT = '\nRavensight Playtester action-selection mode: describe the currently visible screen briefly in text, including each proposed target and its screenshot coordinates, then propose one to three ALTERNATIVE next actions as tool calls. Only one will be executed after a separate selector chooses. These are alternatives, never a sequential plan. Use only observed controls. For a canvas game without accessible DOM controls, include a target label describing the visible control AND screenshot x/y coordinates; the driver uses those coordinates. Never invent selectors such as textbox or button names. When the brief identifies a field as a public player label, use Ravensight Playtester. Credentials or private access codes require human review. When the game is loading or transitioning, propose a bounded wait tool call. Do not place secrets in proposals. After their tool results, observe the resulting state before proposing again.';
 export function describeAction(tool) {
   const input = tool.input || {};
@@ -14,7 +14,7 @@ export function describeAction(tool) {
   };
   return String(descriptions[tool.name] || `${tool.name}: ${JSON.stringify(input)}`).slice(0, 300);
 }
-export async function selectAction({ api, jobId, runId, persona, observation, toolUses, recentActions = [], intentNotes = [] }) {
+export async function selectAction({ api, jobId, runId, persona, observation, toolUses, controller = 'jev', recentActions = [], intentNotes = [] }) {
   const proposals = toolUses.slice(0, 8);
   // Observation does not input anything into the game. Keep it in the existing
   // bounded driver loop; Jev selects game inputs, not whether a frame may load.
@@ -22,6 +22,7 @@ export async function selectAction({ api, jobId, runId, persona, observation, to
     return { status: 'selected', toolId: proposals[0].id, source: 'observation' };
   }
   const body = {
+    controller,
     objective: ('Play the game according to the supplied persona, exploring visible controls and gathering evidence. Developer scope: ' + intentNotes.map(note => note.text || '').join(' ')).slice(0, 1000),
     // The observer already applies the full persona. The selector needs its identity,
     // current visible state and proposed inputs, not a second personality prompt.

package/src/run/aggregate.js

@@ -1,4 +1,5 @@
 import { createHash } from 'node:crypto';
+import { compareControllers, comparisonMarkdown } from './comparison.js';
 import { fillPlaceholders } from './placeholders.js';
 import { writeFile } from 'node:fs/promises';
 import path from 'node:path';
@@ -458,7 +459,9 @@ export async function buildAggregate({
     usage: ledger ? aggregateUsage(ledger) : { usd: 0, by_model: {} }
   };
 
-  return { json, markdown: renderAggregateMarkdown(json) };
+  const comparison = compareControllers(runs);
+  if (comparison) json.controller_comparison = comparison;
+  return { json, markdown: renderAggregateMarkdown(json) + comparisonMarkdown(comparison) };
 }
 
 /**

package/src/run/args.js

@@ -16,6 +16,7 @@ const VALUED = new Map([
   ['--driver', 'driver'],
   ['--driver-config', 'driverConfig'],
   ['--controller', 'controller'],
+  ['--compare-with', 'compareWith'],
   ['--build-url', 'buildUrl'],
   ['--build-dir', 'buildDir'],
   ['--godot-project', 'godotProject'],
@@ -29,6 +30,7 @@ const VALUED = new Map([
 /** Options that are just present or absent. */
 const BOOLEAN = new Map([
   ['--upload-video', 'uploadVideo'],
+  ['--record-video', 'recordVideo'],
   ['--upload-transcript', 'uploadTranscript'],
   ['--dry-upload', 'dryUpload'],
   ['--offline-upload', 'offlineUpload'],

package/src/run/comparison.js

@@ -0,0 +1,32 @@
+// Deterministic comparison of the reports already paid for. No new model call.
+export function compareControllers(runs) {
+  const planned = runs.filter(run => run.controller);
+  const controllers = [...new Set(planned.map(run => run.controller))];
+  if (controllers.length < 2) return null;
+  const rows = planned.map(run => ({
+    run_id: run.runId, persona: run.persona, controller: run.controller,
+    state: run.state, report_available: Boolean(run.report),
+    partial: !run.report || run.report.partial === true || run.state !== 'succeeded',
+    actions_taken: run.report?.actions_taken ?? null,
+  }));
+  const groups = new Map();
+  for (const run of planned) for (const finding of run.report?.findings || []) {
+    if (finding.dropped || !Array.isArray(finding.evidence) || !finding.evidence.length) continue;
+    const key = JSON.stringify([run.persona, finding.category || '', String(finding.title || '').trim().toLowerCase().replace(/\s+/g, ' ')]);
+    if (!groups.has(key)) groups.set(key, { title: finding.title, persona: run.persona, category: finding.category, occurrences: [] });
+    groups.get(key).occurrences.push({ run_id: run.runId, controller: run.controller, evidence: finding.evidence });
+  }
+  const findings = [...groups.values()].map(group => ({ ...group,
+    match: new Set(group.occurrences.map(row => row.controller)).size > 1 ? 'reported_by_multiple_controllers' : 'reported_by_one_controller' }));
+  return { schema_version: '1.0', controllers, status: rows.some(row => row.partial) ? 'inconclusive' : 'reports_complete',
+    limits: ['Controllers use separate game sessions. Server saves and random state are not proven identical.', 'Matches use the same persona, category and exact normalized title. Agreement does not verify a bug; absence does not prove a pass.'], runs: rows, findings };
+}
+export function comparisonMarkdown(comparison) {
+  if (!comparison) return '';
+  const safe = value => String(value ?? '').replace(/[\r\n|]/g, ' ');
+  return '\n\n## Controller comparison\n\n' + comparison.limits.join(' ') + '\n\n' +
+    (comparison.status === 'inconclusive' ? 'At least one playthrough is missing or partial. This comparison is inconclusive.\n\n' : '') +
+    '| Controller | Persona | Run | Report |\n| --- | --- | --- | --- |\n' +
+    comparison.runs.map(row => `| ${safe(row.controller)} | ${safe(row.persona)} | ${safe(row.run_id)} | ${row.partial ? 'Incomplete' : 'Complete'} |`).join('\n') +
+    '\n\n' + comparison.findings.map(group => `- ${safe(group.title)}: ${[...new Set(group.occurrences.map(row => row.controller))].join(', ')} (${group.occurrences.map(row => safe(row.run_id)).join(', ')}).`).join('\n');
+}

package/src/run/controllers.js

@@ -0,0 +1,17 @@
+export const CONTROLLERS = ['standard', 'jev', 'rune'];
+export function controllerPlan(options = {}) {
+  if (options.compareWith && options.controller) throw new Error('Use --controller for one playthrough or --compare-with for a comparison.');
+  if (!options.compareWith && !options.controller) return null;
+  const plan = options.compareWith ? ['standard', ...String(options.compareWith).split(',')] : [options.controller];
+  if (plan.length > 3 || plan.some(x => !CONTROLLERS.includes(x)) || new Set(plan).size !== plan.length) throw new Error('Choose standard, jev or rune once each. --compare-with jev runs standard and Jev separately.');
+  return plan;
+}
+export function quoteMatchesPlan(quote, plan, personas) {
+  return !plan || (Array.isArray(quote.controllers) && JSON.stringify(quote.controllers) === JSON.stringify(plan) && quote.persona_run_count === plan.length * personas.length);
+}
+export function runsMatchPlan(runs, plan, personas) {
+  if (!plan) return true;
+  const actual = runs.filter(run => run.module === 'persona_playtest').map(run => `${run.persona}/${run.controller}`).sort();
+  const expected = personas.flatMap(persona => plan.map(controller => `${persona}/${controller}`)).sort();
+  return JSON.stringify(actual) === JSON.stringify(expected);
+}

package/src/run/drivers/driver.js

@@ -18,7 +18,7 @@ import { createHash } from 'node:crypto';
  * observe()                -> Observation
  * act(action)              -> ActResult
  * screenshot(name, opts)   -> { path, relativePath }
- * stop()                   -> { videoPath?: string|null, framesDir?: string }
+ * stop({ reason })         -> { videoPath?: string|null, recording?: Object }
  * ```
  *
  * Optional, used when present:
@@ -37,10 +37,10 @@ import { createHash } from 'node:crypto';
  *   description in ../tools.js says so, because the model is the one
  *   choosing.
  * - **`stop()` may return no video.** `playwright_web` hands back a
- *   `session.webm` from Playwright's own recorder. `godot_driver` returns
- *   `videoPath: null` and leaves a `frames/` sequence, which the CLI
- *   stitches only when ffmpeg is present. That is what keeps ffmpeg
- *   optional, and it means "no video" is a normal outcome, not a failure.
+ *   `session.webm` from Playwright's own recorder. `godot_driver` encodes
+ *   timestamped viewport frames to `session.mp4` with optional local ffmpeg.
+ *   `recording.status` distinguishes complete, partial, unsupported and failed
+ *   capture; absence of an encoder or rendered viewport never masquerades as video.
  *
  * `Observation.scene`, `.scenes` and `.scripts` are how a finding from a
  * running Godot game can cite a `res://` path; the web driver leaves them