ravensight-playtest 0.1.0

package/src/commands/run.js

@@ -0,0 +1,1015 @@
+import { randomUUID } from 'node:crypto';
+import { mkdir, readFile, stat } from 'node:fs/promises';
+import path from 'node:path';
+import { buildAggregate, writeAggregate } from '../run/aggregate.js';
+import { optionsFrom } from '../run/args.js';
+import { getDeps } from '../run/deps.js';
+import { createDriver } from '../run/drivers/index.js';
+import EXIT from '../run/exit.js';
+import { createHeartbeat } from '../run/heartbeat.js';
+import { createUsageLedger } from '../run/usage.js';
+import { createModelClient, ModelUnavailableError, ProxyError } from '../run/model.js';
+import { jobDir, runDir as runDirFor, JOB_FILES, RUN_FILES } from '../run/paths.js';
+import { runPersona } from '../run/personaLoop.js';
+import * as stateFile from '../run/state.js';
+import { emitRunEvents } from '../run/synthetic.js';
+
+/**
+ * `ravensight-playtest run`: register a job, play it, upload it, finish it.
+ *
+ * The order is load-bearing, and most of it is about money:
+ *
+ * 1. Estimate and confirm BEFORE anything is created. `POST /estimate` answers
+ *    the price and the balance together, so "you are short" is one answer
+ *    rather than a charge that fails.
+ * 2. `POST /jobs` with an `Idempotency-Key`. This is the one request on the
+ *    surface that moves money before it answers, so the key is required, and it
+ *    is written into `state.json` before the runs start: `resume` replays the
+ *    same key, which is why the server hands back the stored job and freshly
+ *    minted tokens rather than charging again.
+ * 3. Play. Each run gets its own heartbeat and its own budget.
+ * 4. Upload, then `complete`, through cli-core's `finalizeRun`. Uploads first,
+ *    because `complete` verifies the manifest before it reads the report: a
+ *    report citing a screenshot that was never uploaded is a 422, by design,
+ *    and `finalizeRun` owns the one repair pass for it.
+ * 5. Aggregate, then `finish`. `finish` settles the job once and issues at most
+ *    one refund, so it is called exactly once, at the end, including after a
+ *    failure.
+ *
+ * Exit codes are spec 17's: 0, 2 out of budget with partial results uploaded, 3
+ * nothing to point at, 4 auth, 5 the proxy never answered, 10 cancelled.
+ */
+
+/** Personas used when the developer names none. Spec 17 defaults to 2. */
+export const DEFAULT_PERSONAS = Object.freeze(['curious-kid', 'grumpy-veteran']);
+
+/** Parallel runs. Each Chromium is 0.5 to 1.5 GB, so 2 is the honest default. */
+export const DEFAULT_CONCURRENCY = 2;
+
+/**
+ * @param {Object} flags what cli-core's commander program hands over
+ * @param {Object} [injected] test seam: deps, createClient, driverFactory, log, confirm
+ * @returns {Promise<number>} process exit code
+ */
+export async function runCommand(flags = {}, injected = {}) {
+  const deps = injected.deps || (await getDeps());
+  const log = injected.log || logger(deps);
+  try {
+    return await run(flags, injected, deps, log);
+  } catch (error) {
+    // Without this a 503 `proxy_disabled` and a ModelUnavailableError both
+    // surface as exit 1, which tells CI nothing. `exitCodeFor` is the one
+    // place that mapping lives.
+    const code = exitCodeFor(error);
+    if (code === EXIT.FAILED) throw error;
+    log(String(error && error.message ? error.message : error));
+    return code;
+  }
+}
+
+/**
+ * @param {Object} flags
+ * @param {Object} injected
+ * @param {Object} deps
+ * @param {(message: string) => void} log
+ * @returns {Promise<number>}
+ */
+async function run(flags, injected, deps, log) {
+  const { options, unknown } = optionsFrom(flags);
+  if (unknown.length > 0) {
+    log(`I do not know these options: ${unknown.join(', ')}. Run "ravensight-playtest run --help".`);
+    return EXIT.ENVIRONMENT;
+  }
+  for (const line of unusedFlagWarnings(options)) log(line);
+
+  const setup = await resolveSetup(deps, options, log);
+  if (setup.code !== undefined) return setup.code;
+  const { context, gameId, repoRoot, apiUrl, cliVersion, api } = setup;
+
+  const driverName = options.driver || (context.config && context.config.driver) || 'playwright_web';
+  const target = resolveTarget(options, context.config, driverName);
+  if (!target) {
+    log('Point the run at a build: --build-url for a web build, or --godot-project for a Godot project.');
+    return EXIT.ENVIRONMENT;
+  }
+
+  // `aggregate_report` is asked for explicitly: it is what buys the run the
+  // `aggregate.*` steps are spendable from, and it is free (the price list has
+  // no unit for it), so asking costs nothing and not asking costs the judged
+  // dedup and success-criteria passes.
+  const modules = ['persona_playtest', 'aggregate_report'];
+  const pack = await deps.packs.get({ api, gameId, modules });
+  const personaSlugs = resolvePersonas(options, context.config, pack);
+  const personas = personaSlugs.map(slug => {
+    const found = (pack.personas || []).find(persona => persona.slug === slug);
+    if (!found) throw new Error(`unknown persona "${slug}"; available: ${(pack.personas || []).map(entry => entry.slug).join(', ')}`);
+    return found;
+  });
+
+  // The same ceiling the server's `personaListRefusal` enforces, checked here
+  // first so a list past it never reaches the network: no estimate, no brief
+  // fetch, nothing charged. `pack.routing` is the clientRouting subset, and an
+  // older cached pack without `max_personas` skips the check rather than
+  // refusing on a field that does not exist yet.
+  const personaRefusal = personaCountRefusal(personaSlugs.length, pack.routing && pack.routing.max_personas);
+  if (personaRefusal) {
+    log(personaRefusal);
+    return EXIT.ENVIRONMENT;
+  }
+
+  // The brief is what a persona is told to respect: scope, known issues, design
+  // intent, the bug definition, the success criteria. Without it every persona
+  // reports findings nobody asked for and re-reports issues the developer
+  // already knows about, so a job that plays the game needs one. The server
+  // agrees and refuses `POST /jobs` with 409 `brief_required`.
+  const brief = await deps.brief.get({ api, gameId });
+  if (!brief) {
+    log('This game has no expectations brief yet, and a persona run needs one: without it the personas do not know what the game is for. Run "ravensight-playtest brief" first.');
+    return EXIT.ENVIRONMENT;
+  }
+  if (brief.status !== 'complete') {
+    log(`The expectations brief for this game is ${brief.status}, not complete. Finish it with "ravensight-playtest brief" and run this again.`);
+    return EXIT.ENVIRONMENT;
+  }
+
+  const estimate = await api.jobs.estimate(gameId, { modules, personas: personaSlugs });
+  log(`${personas.length} persona(s) on ${driverName}: ${formatCents(estimate.estimate_cents)}. Balance: ${formatCents(estimate.balance_cents)}.`);
+  if (estimate.ok === false) {
+    log(`That is more than the balance covers. Top up at ${estimate.topup_url || 'ravensight.io'} and run this again.`);
+    // Nothing is wrong with the request; the account needs money. Spec 17's
+    // table has no code for that, and 3 (environment) is the one that means
+    // "fix something outside this command and try again".
+    return EXIT.ENVIRONMENT;
+  }
+  // Alongside the price, never instead of it: the job is allowed either way,
+  // and the developer decides with the wall clock in front of them, printed
+  // above the confirmation rather than after it is too late to matter.
+  if (estimate.wall_warning) log(estimate.wall_warning);
+  const confirm = injected.confirm || (deps.ui && deps.ui.confirm);
+  if (!options.yes && confirm) {
+    const go = await confirm(`Run ${personas.length} persona(s) for ${formatCents(estimate.estimate_cents)}?`, { yes: options.yes });
+    if (!go) {
+      log('Nothing was charged.');
+      return EXIT.CANCELED;
+    }
+  }
+
+  const idempotencyKey = options.idempotencyKey || randomUUID();
+  let registered;
+  try {
+    registered = await api.jobs.register(gameId, {
+      modules,
+      personas: personaSlugs,
+      driver: driverName,
+      cli_version: cliVersion,
+      confirm_price_cents: estimate.estimate_cents,
+      brief_version: brief.version ?? null,
+      build: { kind: driverName === 'playwright_web' ? 'url' : 'godot_project', ref: target },
+      repo: { commit: (context.config && context.config.commit) || '', dirty: Boolean(context.config && context.config.dirty) },
+      max_actions: Number.isInteger(options.maxActions) ? options.maxActions : null,
+      pack_versions: { pack: pack.pack_version }
+    }, { idempotencyKey });
+  } catch (error) {
+    const handled = explainRegisterRefusal(error, log);
+    if (handled !== null) return handled;
+    throw error;
+  }
+
+  const jobId = registered.job_id;
+  const directory = jobDir(repoRoot, jobId);
+  await mkdir(directory, { recursive: true });
+
+  const journal = (await stateFile.load(repoRoot, jobId)) || stateFile.newState({
+    jobId,
+    gameId,
+    idempotencyKey,
+    cliVersion,
+    packVersion: pack.pack_version,
+    modules,
+    driver: driverName,
+    target,
+    priceCents: registered.price && registered.price.price_cents,
+    options: {
+      maxActions: options.maxActions ?? null,
+      concurrency: options.concurrency ?? DEFAULT_CONCURRENCY,
+      uploadVideo: Boolean(options.uploadVideo),
+      uploadTranscript: Boolean(options.uploadTranscript)
+    }
+  });
+  for (const entry of registered.runs || []) {
+    if (!journal.runs[entry.run_id]) {
+      journal.runs[entry.run_id] = stateFile.newRunEntry({ runId: entry.run_id, module: entry.module, persona: entry.persona });
+    }
+  }
+  await stateFile.save(repoRoot, journal);
+  log(`job ${jobId}${registered.replayed ? ' (replayed)' : ''}: ${formatCents(registered.price && registered.price.price_cents)} charged. ${directory}`);
+
+  return playJob({
+    deps,
+    api,
+    apiUrl,
+    gameId,
+    jobId,
+    repoRoot,
+    runs: registered.runs || [],
+    pack,
+    personas,
+    driverName,
+    target,
+    options,
+    journal,
+    config: context.config,
+    brief,
+    injected,
+    log
+  });
+}
+
+/**
+ * The register refusals that mean something specific to a developer, turned
+ * into a sentence and an exit code. Anything else is rethrown: an error this
+ * function does not recognise should surface as itself.
+ *
+ * @param {any} error
+ * @param {(message: string) => void} log
+ * @returns {number|null} an exit code, or null when the error is not one of these
+ */
+export function explainRegisterRefusal(error, log) {
+  const code = error && error.code;
+  const body = (error && error.body) || {};
+  if (code === 'brief_required') {
+    log('The server refused this job because the game has no complete expectations brief. Run "ravensight-playtest brief" and try again.');
+    return EXIT.ENVIRONMENT;
+  }
+  if (code === 'insufficient_balance') {
+    log(`That is more than the balance covers: ${formatCents(body.shortfall_cents)} short. Top up at ${body.topup_url || 'ravensight.io'} and run this again.`);
+    return EXIT.ENVIRONMENT;
+  }
+  if (code === 'price_changed') {
+    log(`The price changed while you were deciding: it is now ${formatCents(body.estimate_cents)}. Nothing was charged. Run this again to confirm the new price.`);
+    return EXIT.FAILED;
+  }
+  if (code === 'cli_outdated') {
+    log(`This CLI is too old for that server: it needs ${body.min_version || 'a newer version'}. Upgrade with "npm i -g ravensight-playtest".`);
+    return EXIT.ENVIRONMENT;
+  }
+  if (code === 'too_many_active_jobs') {
+    log(`You already have ${body.max_active_jobs || 'the maximum number of'} jobs running on this game. Let one finish, or cancel it in the dashboard.`);
+    return EXIT.FAILED;
+  }
+  if (code === 'charge_in_progress') {
+    log('A charge for this job is still in flight, so the server cannot say yet whether it went through. Wait a moment and run the same command again: the idempotency key makes that safe.');
+    return EXIT.FAILED;
+  }
+  return null;
+}
+
+/**
+ * The client-side half of the server's persona ceiling.
+ *
+ * The server refuses a job over `max_personas` with `error: 'too_many_personas'`
+ * and this exact sentence (`personaListRefusal` in `src/routes/v1/playtest/jobs.js`).
+ * Checking it here too means the developer reads the same words before a single
+ * request goes out, rather than after paying for a round trip to hear it. There
+ * is still no product limit under that ceiling: a large job is allowed and
+ * simply may not finish inside its wall clock, which is what `wall_warning`
+ * says instead.
+ *
+ * @param {number} count
+ * @param {number|null|undefined} maxPersonas the pack's `routing.max_personas`
+ * @returns {string|null}
+ */
+export function personaCountRefusal(count, maxPersonas) {
+  if (!Number.isInteger(maxPersonas) || count <= maxPersonas) return null;
+  return `A job may ask for at most ${maxPersonas} personas, and this one asks for ${count}.`;
+}
+
+/**
+ * Flags this release parses and then does not act on.
+ *
+ * Saying so is the point: a developer who passes `--budget-usd 1` and is not
+ * told it did nothing will believe they capped the spend.
+ *
+ * @param {Object} options
+ * @returns {Array<string>}
+ */
+export function unusedFlagWarnings(options) {
+  const warnings = [];
+  if (options.budgetUsd !== undefined) {
+    warnings.push('--budget-usd is not applied by this release: the model budget is the server\'s, derived from what the job was charged, and it is enforced per call at the proxy.');
+  }
+  if (options.quality !== undefined) {
+    warnings.push('--quality is not applied by this release: the model tier is the server\'s decision, and the CLI is not allowed to name one.');
+  }
+  if (options.offlineUpload !== undefined) {
+    warnings.push('--offline-upload is not applied by this release: uploads already queue and can be drained later with "ravensight-playtest upload <job>".');
+  }
+  if (options.buildDir !== undefined) {
+    warnings.push('--build-dir is not applied by this release: serve the build and pass --build-url instead.');
+  }
+  return warnings;
+}
+
+/**
+ * Play every unfinished run of a registered job, aggregate, finish.
+ *
+ * Shared with `resume`, which arrives at exactly this point by a different
+ * route: it replays the register instead of making one.
+ *
+ * @returns {Promise<number>} an exit code
+ */
+export async function playJob({
+  deps,
+  api,
+  apiUrl,
+  gameId,
+  jobId,
+  repoRoot,
+  runs,
+  pack,
+  personas,
+  driverName,
+  target,
+  options,
+  journal,
+  config,
+  brief = null,
+  injected = {},
+  log = () => {},
+  finish = true
+}) {
+  const createClient = injected.createClient || createModelClient;
+  const makeDriver = injected.driverFactory || createDriver;
+  const token = injected.modelToken || api.token;
+  const client = createClient({ apiUrl, token });
+
+  const directory = jobDir(repoRoot, jobId);
+  const personaRuns = runs.filter(entry => entry.module === 'persona_playtest');
+  const aggregateRun = runs.find(entry => entry.module === 'aggregate_report') || null;
+
+  const results = [];
+  const queue = personaRuns.filter(entry => !stateFile.isRunDone(journal.runs[entry.run_id]));
+  const concurrency = Math.max(1, Math.min(Number(options.concurrency) || DEFAULT_CONCURRENCY, queue.length || 1));
+
+  /**
+   * Runs the process is in the middle of. A Ctrl-C has to move these to
+   * `interrupted` rather than leave them `playing`: the server's sweep would
+   * get there eventually, thirty minutes later, and until then the dashboard
+   * shows a run that is still going and `resume` has nothing to work from.
+   * `interrupted` is the one non-terminal state the transition table can come
+   * back out of, which is exactly what it is for.
+   */
+  const inFlight = new Map();
+  const interrupt = createInterruptHandler({ api, gameId, jobId, repoRoot, journal, inFlight, log, injected });
+
+  try {
+    await Promise.all(Array.from({ length: concurrency }, async () => {
+      while (queue.length > 0 && !interrupt.signalled) {
+        const entry = queue.shift();
+        const persona = personas.find(candidate => candidate.slug === entry.persona) || personas[0];
+        results.push(await playPersonaRun({
+          deps,
+          api,
+          apiUrl,
+          client,
+          config,
+          gameId,
+          jobId,
+          repoRoot,
+          entry,
+          persona,
+          pack,
+          driverName,
+          target,
+          options,
+          journal,
+          inFlight,
+          makeDriver,
+          log
+        }));
+      }
+    }));
+  } finally {
+    interrupt.release();
+  }
+
+  if (interrupt.signalled) {
+    await interrupt.settle();
+    log(`job ${jobId} was interrupted. Everything played so far is in ${directory}; "ravensight-playtest resume ${jobId}" carries on.`);
+    return EXIT.CANCELED;
+  }
+
+  for (const entry of personaRuns) {
+    const done = journal.runs[entry.run_id];
+    if (done && done.completed && !results.some(result => result.runId === entry.run_id)) {
+      results.push({ runId: entry.run_id, persona: entry.persona, state: done.state, quitReason: done.quit_reason, skipped: true });
+    }
+  }
+
+  await aggregatePhase({
+    deps,
+    api,
+    client,
+    gameId,
+    jobId,
+    repoRoot,
+    directory,
+    aggregateRun,
+    personaRuns,
+    pack,
+    brief,
+    driverName,
+    options,
+    journal,
+    log
+  });
+
+  const budgetHit = results.some(result => result.state === 'budget_exceeded');
+  const canceled = results.some(result => result.state === 'canceled');
+  const anySucceeded = results.some(result => result.state === 'succeeded');
+
+  if (finish) {
+    const finished = await api.jobs.finish(gameId, jobId, {
+      state: budgetHit && !anySucceeded ? 'budget_exceeded' : undefined,
+      reason: canceled ? 'cancelled by the developer' : ''
+    });
+    journal.finished = true;
+    await stateFile.save(repoRoot, journal);
+    log(`job ${jobId} ${finished.job ? finished.job.state : 'finished'}: refunded ${formatCents(finished.refunded_cents)}.`);
+  }
+  log(`Reports: ${directory}`);
+
+  if (canceled) return EXIT.CANCELED;
+  if (budgetHit) return EXIT.BUDGET;
+  return anySucceeded ? EXIT.OK : EXIT.FAILED;
+}
+
+/**
+ * One persona, start to finish: play, upload, complete.
+ *
+ * @returns {Promise<{runId: string, persona: string, state: string, quitReason: string}>}
+ */
+export async function playPersonaRun({
+  deps,
+  api,
+  apiUrl,
+  client,
+  config,
+  gameId,
+  jobId,
+  repoRoot,
+  entry,
+  persona,
+  pack,
+  driverName,
+  target,
+  options,
+  journal,
+  inFlight = null,
+  makeDriver,
+  log
+}) {
+  const runId = entry.run_id;
+  const directory = runDirFor(repoRoot, jobId, runId);
+  await mkdir(directory, { recursive: true });
+  if (inFlight) inFlight.set(runId, { state: 'queued' });
+
+  const skill = skillFor(pack, 'persona_playtest');
+  const maxActions = resolveMaxActions(options, persona, pack);
+  // Headless is right for a browser and wrong for Godot: a headless Godot
+  // routes no GUI input, so it is a smoke-test mode rather than a way to play.
+  const headless = driverName === 'godot_driver' ? options.headless === true : options.headed !== true;
+  const driver = await makeDriver(driverName, { headless });
+  // The ledger is created here rather than inside the loop so the heartbeat
+  // can feed it the server's own figure for what this run has spent. Prices
+  // are server side only, so that answer is the only real one.
+  const ledger = createUsageLedger();
+  const heartbeat = createHeartbeat({
+    api,
+    gameId,
+    jobId,
+    runId,
+    onCancel: () => log(`run ${runId}: the job was cancelled; stopping after this turn.`),
+    onUsage: usage => ledger.recordServerUsage(usage),
+    // Every beat writes the checkpoint into the journal. That is what lets a
+    // resume tell a run that got three actions in from one that got thirty:
+    // without it the journal only ever says 0 until the run ends, and a
+    // resume has no way to prefer finalising over replaying.
+    onBeat: async beat => {
+      if (!beat) return;
+      try {
+        await stateFile.patchRun(repoRoot, journal, runId, {
+          state: 'playing',
+          checkpoint_step: Number(beat.checkpointStep) || 0,
+          actions_taken: Number(beat.actionsTaken) || 0
+        });
+      } catch {
+        // A journal write that fails must not kill a paid run; the next beat
+        // tries again five seconds later.
+      }
+    },
+    onError: () => {}
+  });
+  heartbeat.start();
+
+  let outcome;
+  try {
+    if (inFlight) inFlight.set(runId, { state: 'launching' });
+    outcome = await runPersona({
+      client,
+      api,
+      gameId,
+      jobId,
+      runId,
+      persona,
+      driver,
+      driverName,
+      target,
+      runDirectory: directory,
+      skill,
+      brief: pack.brief || null,
+      intentNotes: pack.intent_notes || [],
+      schemas: pack.schemas || [],
+      maxActions,
+      heartbeat,
+      ledger,
+      recordVideo: Boolean(options.uploadVideo),
+      log: message => log(`run ${runId}: ${message}`)
+    });
+  } catch (error) {
+    heartbeat.stop();
+    if (inFlight) inFlight.delete(runId);
+    const failed = error.outcome || { state: 'failed', quitReason: 'error', actionsTaken: 0, report: null, reportLint: [], reachedState: 'playing' };
+    await stateFile.patchRun(repoRoot, journal, runId, { state: failed.state, error: String(error && error.message), quit_reason: failed.quitReason });
+    await deliver({ deps, api, gameId, jobId, runId, directory, outcome: failed, options, log });
+    log(`run ${runId} (${persona.slug}) failed: ${String(error && error.message)}`);
+    return { runId, persona: persona.slug, state: failed.state, quitReason: failed.quitReason, error };
+  }
+  heartbeat.stop();
+  if (inFlight) inFlight.delete(runId);
+
+  await stateFile.patchRun(repoRoot, journal, runId, {
+    state: outcome.state,
+    actions_taken: outcome.actionsTaken,
+    quit_reason: outcome.quitReason,
+    report_written: Boolean(outcome.report),
+    budget_warned: outcome.budgetWarned
+  });
+
+  await deliver({ deps, api, gameId, jobId, runId, directory, outcome, options, log, journal, repoRoot });
+
+  if (outcome.report) {
+    const emitted = await emitRunEvents({
+      apiUrl,
+      ingestKey: (config && config.ingest_key) || process.env.RAVENSIGHT_INGEST_KEY || null,
+      playtestToken: entry.playtest_token,
+      context: { jobId, runId, persona: persona.slug, gameVersion: (config && config.game_version) || 'unknown' },
+      report: outcome.report,
+      driver: driverName,
+      log: message => log(`run ${runId}: ${message}`)
+    });
+    if (emitted.reason) log(`run ${runId}: ${emitted.reason}`);
+  }
+
+  log(`run ${runId} (${persona.slug}) ${outcome.state}: ${outcome.actionsTaken} actions, ${outcome.quitReason}.`);
+  return { runId, persona: persona.slug, state: outcome.state, quitReason: outcome.quitReason };
+}
+
+/**
+ * Upload the run directory and complete the run, through cli-core's
+ * `finalizeRun` so the 422 `upload_unverified` repair pass lives in one place.
+ *
+ * A budget-capped or cancelled run sends no `report`: the route accepts that,
+ * and it is the honest answer. Inventing a sentiment score nobody gave so a
+ * placeholder report would validate would put a fabricated verdict in front of
+ * the developer.
+ *
+ * A refused delivery does not take the job down. `finish` still settles it, the
+ * artifacts are on disk, and `resume` can retry.
+ */
+async function deliver({ deps, api, gameId, jobId, runId, directory, outcome, options, log, journal, repoRoot }) {
+  const body = {
+    state: outcome.state,
+    reason: outcome.quitDetail || '',
+    actions_taken: outcome.actionsTaken,
+    quit_reason: outcome.quitReason
+  };
+  if (outcome.report) body.report = outcome.report;
+  // `report_md` is what the server lints for its four required sections. Not
+  // sending it means `report_lint` is silently empty on every run, which reads
+  // as "the markdown is fine" rather than "nobody looked".
+  if (outcome.reportMarkdown) body.report_md = outcome.reportMarkdown;
+
+  if (options.dryUpload) {
+    const listing = await listUploadable(directory, options);
+    log(`run ${runId}: --dry-upload, nothing was sent. Would upload:\n${listing.map(item => `  ${item.path} (${item.size} bytes)`).join('\n')}`);
+    return null;
+  }
+
+  // The server's table allows `uploading` only from `reporting`, so asking for
+  // it from `playing` (a run that died before it wrote anything) is a 409 with
+  // a confusing message. The run says where it got to; believe it.
+  if (outcome.reachedState === 'reporting') {
+    try {
+      await api.runs.transition(gameId, jobId, runId, 'uploading', { reason: '' });
+    } catch (error) {
+      // A refused transition is not worth abandoning the artifacts over: the
+      // upload and the complete below are what actually matter.
+      log(`run ${runId}: could not move to uploading (${String(error && error.message)}).`);
+    }
+  }
+
+  try {
+    const answer = await deps.finalizeRun(directory, {
+      api,
+      gameId,
+      jobId,
+      runId,
+      includeVideo: Boolean(options.uploadVideo),
+      includeTranscript: Boolean(options.uploadTranscript),
+      body
+    });
+    if (journal) await stateFile.patchRun(repoRoot, journal, runId, { uploaded: true, completed: true });
+    if (answer && answer.dropped_findings > 0) {
+      log(`run ${runId}: the server dropped ${answer.dropped_findings} finding(s) with no usable evidence.`);
+    }
+    if (answer && Array.isArray(answer.report_lint) && answer.report_lint.length > 0) {
+      log(`run ${runId}: report lint: ${answer.report_lint.join('; ')}`);
+    }
+    return answer;
+  } catch (error) {
+    log(`run ${runId}: upload or complete was refused (${String(error && error.message)}). The artifacts are in the run directory and "ravensight-playtest resume" can retry it.`);
+    return null;
+  }
+}
+
+/**
+ * Build the aggregate, and drive the `aggregate_report` run's own state machine
+ * while doing it.
+ *
+ * That run is what makes the judged passes possible: `aggregate.*` steps are
+ * only spendable from a run whose module is `aggregate_report`, so it is
+ * launched, played, and completed like any other run rather than treated as
+ * bookkeeping. Without one (a job registered before the module existed) the
+ * aggregate is still built, locally, and says so.
+ */
+async function aggregatePhase({
+  deps,
+  api,
+  client,
+  gameId,
+  jobId,
+  repoRoot,
+  directory,
+  aggregateRun,
+  personaRuns,
+  pack,
+  brief,
+  driverName,
+  options,
+  journal,
+  log
+}) {
+  const runId = aggregateRun ? aggregateRun.run_id : null;
+  const ledger = createUsageLedger();
+  const heartbeat = runId
+    ? createHeartbeat({ api, gameId, jobId, runId, onUsage: usage => ledger.recordServerUsage(usage), onError: () => {} })
+    : null;
+
+  let reached = 'queued';
+  const advance = async state => {
+    if (!runId) return;
+    try {
+      await api.runs.transition(gameId, jobId, runId, state, { reason: '' });
+      reached = state;
+    } catch (error) {
+      log(`aggregate run ${runId}: could not move to ${state} (${String(error && error.message)}).`);
+    }
+  };
+
+  await advance('launching');
+  await advance('playing');
+  if (heartbeat) heartbeat.start();
+
+  // The aggregate reads reports off disk, so a resumed job whose earlier runs
+  // were played by a previous process is covered too.
+  const reports = await loadRunReports(repoRoot, jobId, personaRuns);
+  let aggregate;
+  try {
+    aggregate = await buildAggregate({
+      client,
+      jobId,
+      gameId,
+      aggregateRunId: runId,
+      runs: reports,
+      brief,
+      capabilityNotes: capabilityNotesFor(pack, driverName),
+      intentNotes: pack.intent_notes || [],
+      ledger,
+      log
+    });
+  } finally {
+    if (heartbeat) {
+      await heartbeat.beat();
+      heartbeat.stop();
+    }
+  }
+
+  await advance('reporting');
+  await writeAggregate(directory, aggregate);
+  journal.aggregate = {
+    state: 'written',
+    generated_from: aggregate.json.provenance.generated_from,
+    run_id: runId
+  };
+  await stateFile.save(repoRoot, journal);
+  if (!runId) {
+    log('The aggregate was built locally: this job has no aggregate_report run to spend a model turn from, so the dedup and judgement passes were skipped.');
+  }
+
+  const published = await publishAggregate({ deps, api, gameId, jobId, repoRoot, directory, options, log });
+  if (published === 'finished') reached = 'succeeded';
+  else if (published) await advance('uploading');
+
+  if (runId && reached !== 'succeeded') {
+    try {
+      await api.runs.complete(gameId, jobId, runId, {
+        state: reached === 'uploading' ? 'succeeded' : 'failed',
+        actions_taken: 0,
+        quit_reason: reached === 'uploading' ? 'goal_reached' : 'error',
+        reason: reached === 'uploading' ? '' : 'the aggregate was not published'
+      });
+    } catch (error) {
+      log(`aggregate run ${runId}: complete was refused (${String(error && error.message)}).`);
+    }
+  }
+  return aggregate;
+}
+
+/**
+ * Publish the aggregate and tell the server it is there.
+ *
+ * `POST /jobs/:jobId/aggregate/complete` verifies the markdown and the JSON
+ * against the sizes and sha256s it is given, so all four come from the bytes
+ * that were uploaded.
+ *
+ * @returns {Promise<false|'published'|'finished'>} false when nothing was
+ *   sent, 'published' when the server has it, 'finished' when it also closed
+ *   the aggregate run
+ */
+async function publishAggregate({ deps, api, gameId, jobId, repoRoot, directory, options, log }) {
+  if (options.dryUpload) {
+    log('The aggregate was not published: --dry-upload.');
+    return false;
+  }
+  if (!deps.publishJobFiles) {
+    log('The aggregate stayed local: this build has no job-level upload.');
+    return false;
+  }
+  try {
+    const digests = await deps.publishJobFiles({
+      api,
+      gameId,
+      jobId,
+      dir: directory,
+      files: [JOB_FILES.aggregateMd, JOB_FILES.aggregateJson],
+      repoRoot
+    });
+    const markdown = digests[JOB_FILES.aggregateMd];
+    const json = digests[JOB_FILES.aggregateJson];
+    // Both halves, not just the markdown: the server verifies each one it
+    // will later serve, and a missing pair reads as "nothing declared" and
+    // refuses the whole aggregate (found in the first production run).
+    const answer = await api.jobs.aggregateComplete(gameId, jobId, {
+      md_size: markdown.size,
+      md_sha256: markdown.sha256,
+      json_size: json.size,
+      json_sha256: json.sha256
+    });
+    // The server finishes the aggregate run itself when it accepts the
+    // report, and says so in `run`. A caller that then tries to move the
+    // run to `uploading` and complete it gets two refusals for a run that
+    // is already done.
+    const run = answer && answer.run;
+    return run && run.state === 'succeeded' ? 'finished' : 'published';
+  } catch (error) {
+    log(`The aggregate could not be published (${String(error && error.message)}). It is on disk in ${directory}.`);
+    return false;
+  }
+}
+
+/**
+ * The Ctrl-C path.
+ *
+ * One handler for the whole job rather than one per run, registered once and
+ * removed in a `finally`, because a handler left behind by a command that has
+ * returned is a handler that fires for the next one. A second signal is left to
+ * the default behaviour: someone pressing Ctrl-C twice means it now, and
+ * holding the process open to be tidy would be the wrong answer.
+ *
+ * @returns {{signalled: boolean, settle: () => Promise<void>, release: () => void}}
+ */
+export function createInterruptHandler({ api, gameId, jobId, repoRoot, journal, inFlight, log, injected = {} }) {
+  const signals = injected.signals || ['SIGINT', 'SIGTERM'];
+  const target = injected.process || process;
+  const state = { signalled: false, signal: null };
+
+  const onSignal = signal => {
+    if (state.signalled) return;
+    state.signalled = true;
+    state.signal = signal;
+    log(`\n${signal}: finishing the turn in flight, then stopping. Nothing is lost; "ravensight-playtest resume ${jobId}" carries on.`);
+    // Hand the signal back to the default behaviour, so a second press kills
+    // the process rather than being swallowed by a handler that is now done.
+    for (const name of signals) target.removeListener(name, onSignal);
+  };
+
+  for (const name of signals) target.on(name, onSignal);
+
+  return {
+    get signalled() {
+      return state.signalled;
+    },
+
+    /**
+     * Move every run that was mid-flight to `interrupted` and flush the
+     * journal, so the server and the file on disk agree with each other.
+     */
+    async settle() {
+      for (const [runId] of inFlight) {
+        try {
+          await api.runs.transition(gameId, jobId, runId, 'interrupted', { reason: `interrupted by ${state.signal || 'a signal'}` });
+        } catch (error) {
+          log(`run ${runId}: could not be marked interrupted (${String(error && error.message)}). The server's sweep will do it.`);
+        }
+        try {
+          await stateFile.patchRun(repoRoot, journal, runId, { state: 'interrupted' });
+        } catch {
+          // The journal write is best effort at this point; the server's own
+          // run state is what `resume` reconciles against anyway.
+        }
+      }
+      inFlight.clear();
+      await stateFile.save(repoRoot, journal);
+    },
+
+    release() {
+      for (const name of signals) target.removeListener(name, onSignal);
+    }
+  };
+}
+
+/**
+ * Resolve the context, the game and the credential, or answer with an exit
+ * code. Shared with `profile`.
+ *
+ * @returns {Promise<Object>}
+ */
+export async function resolveSetup(deps, options, log) {
+  let context;
+  try {
+    context = await deps.createContext({ apiUrl: options.apiUrl, repoRoot: options.repo, requireAuth: true });
+  } catch (error) {
+    log(`You are not signed in. Run "ravensight-playtest login" first. (${String(error && error.message)})`);
+    return { code: EXIT.AUTH };
+  }
+  if (!context.token && !options.token) {
+    log('You are not signed in. Run "ravensight-playtest login" first.');
+    return { code: EXIT.AUTH };
+  }
+
+  let gameId;
+  try {
+    gameId = deps.resolveGameId
+      ? deps.resolveGameId(options, context.config)
+      : (options.game || (context.config && context.config.game_id));
+  } catch (error) {
+    log(String(error && error.message));
+    return { code: EXIT.ENVIRONMENT };
+  }
+  if (!gameId) {
+    log('This repository is not linked to a game. Run "ravensight-playtest init --game <gameId>" first.');
+    return { code: EXIT.ENVIRONMENT };
+  }
+
+  return {
+    context,
+    api: context.api,
+    gameId,
+    apiUrl: context.apiUrl,
+    repoRoot: path.resolve(context.repoRoot || options.repo || process.cwd()),
+    cliVersion: deps.cliVersion || context.api.cliVersion || '0.0.0'
+  };
+}
+
+async function loadRunReports(repoRoot, jobId, personaRuns) {
+  const out = [];
+  for (const entry of personaRuns) {
+    const file = path.join(runDirFor(repoRoot, jobId, entry.run_id), RUN_FILES.reportJson);
+    try {
+      out.push({ runId: entry.run_id, persona: entry.persona, state: 'succeeded', report: JSON.parse(await readFile(file, 'utf8')) });
+    } catch {
+      out.push({ runId: entry.run_id, persona: entry.persona, state: 'failed', report: null });
+    }
+  }
+  return out;
+}
+
+async function listUploadable(directory, options) {
+  const names = [RUN_FILES.reportMd, RUN_FILES.reportJson, RUN_FILES.usageJson];
+  if (options.uploadTranscript) names.push(RUN_FILES.transcript);
+  if (options.uploadVideo) names.push(RUN_FILES.video);
+  const out = [];
+  for (const name of names) {
+    try {
+      const info = await stat(path.join(directory, name));
+      out.push({ path: name, size: info.size });
+    } catch {
+      // Not every run produces every file; a missing one simply is not
+      // uploaded, and saying so is the point of --dry-upload.
+    }
+  }
+  return out;
+}
+
+/**
+ * What this job could NOT test, which is what stops a reader over-trusting it.
+ */
+export function capabilityNotesFor(pack, driverName) {
+  const notes = [];
+  if (driverName === 'playwright_web') {
+    notes.push('Played through a web build with keyboard and mouse at human speed. Reflex, aim and physics feel were not tested.');
+  }
+  if (!pack.code_brief) notes.push('No codebase model was available, so no finding cites a source path.');
+  return notes;
+}
+
+export function skillFor(pack, moduleName) {
+  const entry = (pack.skills || []).find(skill => skill.module === moduleName);
+  if (!entry) throw new Error(`the content pack has no ${moduleName} skill; run a pack sync`);
+  return entry.content;
+}
+
+export function resolvePersonas(options, config, pack) {
+  if (Array.isArray(options.personas) && options.personas.length > 0) return options.personas;
+  if (typeof options.personas === 'string' && options.personas.trim()) {
+    return options.personas.split(',').map(slug => slug.trim()).filter(Boolean);
+  }
+  if (config && Array.isArray(config.personas) && config.personas.length > 0) return config.personas;
+  const available = (pack.personas || []).map(persona => persona.slug);
+  const defaults = DEFAULT_PERSONAS.filter(slug => available.includes(slug));
+  return defaults.length > 0 ? defaults : available.slice(0, 2);
+}
+
+/**
+ * The action budget is the smallest of the three ceilings that exist: what the
+ * developer asked for, the persona's own patience, and what routing.json allows
+ * per run (`max_actions`, which arrives in the pack's client routing subset).
+ */
+export function resolveMaxActions(options, persona, pack) {
+  const candidates = [
+    Number.isInteger(options.maxActions) ? options.maxActions : null,
+    Number(persona && persona.patience_actions) || null,
+    Number(pack && pack.routing && pack.routing.max_actions) || null
+  ].filter(value => Number.isFinite(value) && value > 0);
+  return candidates.length > 0 ? Math.min(...candidates) : 25;
+}
+
+export function resolveTarget(options, config, driverName) {
+  const build = (config && config.build) || {};
+  if (driverName === 'playwright_web') return options.buildUrl || (build.kind === 'url' ? build.ref : null) || null;
+  if (driverName === 'godot_driver') return options.godotProject || (build.kind === 'godot_project' ? build.ref : null) || null;
+  return options.cli || null;
+}
+
+function logger(deps) {
+  if (deps && deps.ui && typeof deps.ui.info === 'function') return message => deps.ui.info(message);
+  return message => console.log(message);
+}
+
+export function formatCents(cents) {
+  if (!Number.isFinite(Number(cents))) return 'unknown';
+  return `$${(Number(cents) / 100).toFixed(2)}`;
+}
+
+/**
+ * Map a thrown error onto an exit code, for whoever wires this into commander.
+ *
+ * @param {any} error
+ * @returns {number}
+ */
+export function exitCodeFor(error) {
+  if (error instanceof ModelUnavailableError) return EXIT.MODEL;
+  if (error instanceof ProxyError) {
+    if (error.isBudget) return EXIT.BUDGET;
+    if (error.isClosed) return EXIT.CANCELED;
+    if (error.isAuth) return EXIT.AUTH;
+    // The kill switch, or a server with no platform key. Nothing the developer
+    // did wrong and nothing a retry fixes, but it IS a model-access failure,
+    // which is what 5 means.
+    if (error.status === 503) return EXIT.MODEL;
+  }
+  return EXIT.FAILED;
+}
+
+export default runCommand;