ravensight-playtest 0.5.2 → 0.7.0

package/src/commands/init.js

@@ -6,6 +6,8 @@ import { withAuthHandling } from '../auth/session.js';
 import { paths } from '../paths.js';
 import { ui } from '../ui/index.js';
 import { CliError, ExitCode } from '../errors.js';
+import { recommendConcurrency } from '../detect.js';
+import { isGodotProject } from '../run/drivers/godot-project.js';
 
 const GITIGNORE_LINE = '.ravensight/jobs/';
 
@@ -66,7 +68,7 @@ export async function init(flags = {}, deps = {}) {
   // Every API call in this command goes through the one 401 handler, so a
   // refused credential is cleared and explained here the same way it is
   // everywhere else.
-  return withAuthHandling(context, () => linkRepo(context, flags, repoRoot, deps.choose || ui.choose));
+  return withAuthHandling(context, () => linkRepo(context, flags, repoRoot, deps.choose || ui.choose, deps.recommend || recommendConcurrency));
 }
 
 /**
@@ -76,7 +78,7 @@ export async function init(flags = {}, deps = {}) {
  * @param {(question: string, labels: Array<string>) => Promise<number|null>} choose
  *   the picker; answers null when there is no terminal to ask on
  */
-async function linkRepo(context, flags, repoRoot, choose) {
+async function linkRepo(context, flags, repoRoot, choose, recommend) {
   const whoami = await context.api.cli.whoami();
 
   const games = whoami.games || [];
@@ -130,8 +132,20 @@ async function linkRepo(context, flags, repoRoot, choose) {
     );
   }
 
+  // What check would say: a repository with a project.godot is played with
+  // the Godot driver, and the concurrency is the one this machine can take.
+  // 0.5.0 wrote playwright_web and 2 right after check said Godot and 1.
+  const godot = Boolean(await isGodotProject(repoRoot));
+  const machine = recommend();
   const config = await saveConfig(
-    { ...(existing || {}), ...defaultConfig(gameId), api_url: context.apiUrl },
+    {
+      ...(existing || {}),
+      ...defaultConfig(gameId, {
+        concurrency: machine.recommended,
+        ...(godot ? { driver: 'godot_driver', build: { kind: 'godot_project', ref: '.' } } : {})
+      }),
+      api_url: context.apiUrl
+    },
     { repoRoot }
   );
   const gitignore = await ensureGitignore(repoRoot);
@@ -144,6 +158,8 @@ async function linkRepo(context, flags, repoRoot, choose) {
   const name = (games.find(game => game.gameId === gameId) || {}).name || gameId;
   ui.ok(`Linked this repo to ${name} (${gameId}).`);
   ui.info(`Wrote ${paths.configFile({ repoRoot })}`);
+  if (godot) ui.info('This is a Godot project, so runs use godot_driver on this directory.');
+  ui.info(`Concurrency ${config.concurrency}, from this machine (${machine.cpus} logical cpus, ${machine.freeMemGb} GB free).`);
   if (gitignore === 'added') ui.info(`Added ${GITIGNORE_LINES.join(' and ')} to .gitignore`);
   if (gitignore === 'absent') ui.warn(`No .gitignore here. Add ${GITIGNORE_LINES.join(' and ')} to whatever ignores build output.`);
   ui.blank();

package/src/commands/lists.js

@@ -0,0 +1,206 @@
+import { createContext } from '../api/index.js';
+import { ApiError } from '../api/errors.js';
+import { resolveGameId } from '../config.js';
+import { withAuthHandling } from '../auth/session.js';
+import { CliError, ExitCode } from '../errors.js';
+import { ui } from '../ui/index.js';
+
+/**
+ * What `punch` and `tape` share: the game, the client, the refusals in plain
+ * words, and how an item is printed.
+ *
+ * The punch list holds what has to be finished before handover, closed one at
+ * a time. Blue tape holds small first-impression annoyances, captured fast and
+ * promoted to the punch list when they matter. Both live on the server under
+ * the game's Labs lists, so the dashboard and this CLI read the same items.
+ */
+
+export const LIST_NAMES = Object.freeze({ punch: 'punch list', tape: 'blue tape list' });
+export const STATUSES = Object.freeze(['open', 'done', 'dropped', 'all']);
+export const ITEM_ID = /^li_[A-Za-z0-9_-]+$/;
+export const LIMIT = 500;
+
+/**
+ * Run `fn` with the client and the game this repo is linked to (or `--game`).
+ * @param {Object} flags
+ * @param {Object} deps test seam: context
+ * @param {(setup: {api: Object, gameId: string}) => Promise<number>} fn
+ * @returns {Promise<number>}
+ */
+export async function withList(flags, deps, fn) {
+  const context = deps.context || await createContext({ apiUrl: flags.apiUrl, repoRoot: flags.repoRoot, requireAuth: true });
+  return withAuthHandling(context, async () => {
+    const gameId = resolveGameId(flags, context.config);
+    return fn({ api: context.api, gameId });
+  });
+}
+
+/**
+ * A refusal from the lists routes as a sentence a person can act on. Anything
+ * this does not recognise is handed back unchanged for `report` to print.
+ *
+ * @param {unknown} error
+ * @param {{list: string, itemId?: string|null}} where
+ * @returns {unknown}
+ */
+export function listError(error, { list, itemId = null }) {
+  if (!(error instanceof ApiError) && !(error && typeof error.status === 'number' && typeof error.code === 'string')) return error;
+  const name = LIST_NAMES[list] || 'list';
+  if (error.code === 'labs_not_enabled') {
+    return new CliError(
+      `The ${name} is part of Ravensight Labs (beta). Labs is not on for this game.`,
+      ExitCode.ENVIRONMENT,
+      { hint: 'Turn Labs on from the Labs page in the dashboard. Nothing was changed.' }
+    );
+  }
+  if (error.code === 'item_closed') {
+    return new CliError(`${itemId || 'That item'} is closed. Reopen it first, then promote it.`, ExitCode.FAILURE);
+  }
+  if (error.code === 'quota_exceeded') {
+    return new CliError('This account has no AI units left. Add funds, then try again. Nothing was charged.', ExitCode.FAILURE);
+  }
+  if (error.code === 'labs_list_attempts') {
+    return new CliError('Too many drafts came back empty this hour. Try again later. Nothing was charged.', ExitCode.FAILURE);
+  }
+  if (error.code === 'ai_unavailable') {
+    return new CliError('The AI service is not available right now. Try again later. Nothing was charged.', ExitCode.FAILURE);
+  }
+  if (error.code === 'list_full') {
+    return new CliError(
+      `The ${name} already has ${LIMIT} open items. Close or drop some, then add this one.`,
+      ExitCode.FAILURE
+    );
+  }
+  if (error.status === 404) {
+    if (itemId) {
+      return new CliError(
+        `No item ${itemId} is on this game's ${name}.`,
+        ExitCode.FAILURE,
+        { hint: `Run "ravensight-playtest ${list} list --status all" to see the item ids.` }
+      );
+    }
+    return new CliError(
+      `This game has no ${name} that this token can see.`,
+      ExitCode.FAILURE,
+      { hint: 'Run "ravensight-playtest check" to confirm the game is linked. An older server has no lists yet.' }
+    );
+  }
+  return error;
+}
+
+/**
+ * Call the server and turn its refusals into sentences.
+ * @template T
+ * @param {() => Promise<T>} call
+ * @param {{list: string, itemId?: string|null}} where
+ * @returns {Promise<T>}
+ */
+export async function guarded(call, where) {
+  try {
+    return await call();
+  } catch (error) {
+    throw listError(error, where);
+  }
+}
+
+/**
+ * Refuse an id that cannot be an item before asking the server about it.
+ * @param {string} itemId
+ */
+export function requireItemId(itemId) {
+  const value = String(itemId || '').trim();
+  if (!ITEM_ID.test(value)) {
+    throw new CliError(`"${value}" is not an item id. Item ids start with li_.`, ExitCode.FAILURE, {
+      hint: 'Run "ravensight-playtest punch list" or "ravensight-playtest tape list" to see them.'
+    });
+  }
+  return value;
+}
+
+/**
+ * Refuse empty text here rather than spend a round trip on it.
+ * @param {string} text
+ */
+export function requireText(text) {
+  const value = String(text || '').trim();
+  if (!value) throw new CliError('Say what the item is, in quotes.', ExitCode.FAILURE);
+  return value;
+}
+
+/** @param {string|undefined} status */
+export function requireStatus(status) {
+  if (status === undefined || status === null) return 'open';
+  if (!STATUSES.includes(status)) {
+    throw new CliError(`--status takes ${STATUSES.join(', ')}.`, ExitCode.FAILURE);
+  }
+  return status;
+}
+
+/** Where an item came from, in a few words. */
+export function sourceWords(source) {
+  if (!source || !source.kind || source.kind === 'manual') return 'manual';
+  const kind = String(source.kind).replace(/_/g, ' ');
+  return source.ref ? `${kind} ${source.ref}` : kind;
+}
+
+/** The plain-language hints under an item. A hint, never a block. */
+export function printHints(item) {
+  const issues = item && item.plain_check && Array.isArray(item.plain_check.issues) ? item.plain_check.issues : [];
+  for (const issue of issues) {
+    if (issue && issue.message) ui.info(`  hint: ${issue.message}`);
+  }
+  return issues.length;
+}
+
+/**
+ * Print a list answer as a table, or say it is empty.
+ * @param {Object} answer `{ items, counts }`
+ * @param {string} list
+ * @param {string} status
+ */
+export function printList(answer, list, status) {
+  const items = answer && Array.isArray(answer.items) ? answer.items : [];
+  const name = LIST_NAMES[list];
+  if (items.length === 0) {
+    ui.info(status === 'all' ? `The ${name} is empty.` : `The ${name} has no ${status} items.`);
+  } else {
+    ui.table(items.map(item => ({
+      item_id: item.item_id,
+      status: item.status,
+      text: item.text,
+      source: sourceWords(item.source) + (item.promoted_to ? `, now ${item.promoted_to}` : '') + (item.fix_job_id ? `, fix ${item.fix_job_id}` : '')
+    })), [
+      { key: 'item_id', label: 'ITEM' },
+      { key: 'status', label: 'STATUS' },
+      { key: 'text', label: 'TEXT' },
+      { key: 'source', label: 'SOURCE' }
+    ]);
+  }
+  const counts = answer && answer.counts;
+  if (counts) {
+    ui.blank();
+    ui.info(`${counts.open ?? 0} open, ${counts.done ?? 0} done, ${counts.dropped ?? 0} dropped.`);
+  }
+}
+
+/**
+ * Find one item by id, for `fix li_...`. The punch list first, because that is
+ * where a finding to fix usually sits, then blue tape on a 404.
+ *
+ * @param {Object} api
+ * @param {string} gameId
+ * @param {string} itemId
+ * @returns {Promise<{item: Object, list: string}|null>}
+ */
+export async function findListItem(api, gameId, itemId) {
+  for (const list of ['punch', 'tape']) {
+    try {
+      const answer = await api.labs.lists.get(gameId, list, itemId);
+      if (answer && answer.item) return { item: answer.item, list };
+    } catch (error) {
+      if (error && error.status === 404) continue;
+      throw listError(error, { list, itemId });
+    }
+  }
+  return null;
+}

package/src/commands/profile.js

@@ -1,12 +1,13 @@
 import { randomUUID } from 'node:crypto';
 import { mkdir } from 'node:fs/promises';
 import { optionsFrom, unknownOptionNotes } from '../run/args.js';
+import { settleFailureLine } from '../run/settle.js';
 import { getDeps } from '../run/deps.js';
 import EXIT from '../run/exit.js';
 import { createHeartbeat } from '../run/heartbeat.js';
 import { createModelClient, ModelUnavailableError, ProxyError } from '../run/model.js';
 import { jobDir } from '../run/paths.js';
-import { runGameProfile, writeCapabilityReport } from '../run/profile.js';
+import { runGameProfile, writeCapabilityReport, writeProfileFailure } from '../run/profile.js';
 import * as stateFile from '../run/state.js';
 import { createUsageLedger, writeUsageFile } from '../run/usage.js';
 import { formatCents, resolveSetup } from './run.js';
@@ -64,6 +65,11 @@ async function profile(flags, injected, deps, log) {
   const { context, api, gameId, apiUrl, repoRoot, cliVersion } = setup;
 
   const modules = ['game_profile'];
+  // The profile is a read of the repository: --driver, --godot-project and
+  // --godot-path are accepted for symmetry with run and change nothing here.
+  if (options.driver || options.godotProject || options.godotPath) {
+    log('profile reads the repository and does not launch the game, so --driver, --godot-project and --godot-path are not used here. A persona run (run) is what plays the game.');
+  }
   const pack = await deps.packs.get({ api, gameId, modules });
   // A profile job is the one shape that does not need a brief, so a missing one
   // is not an error here. It is still read when there is one: a brief that
@@ -186,7 +192,8 @@ async function profile(flags, injected, deps, log) {
       actions_taken: turns,
       quit_reason: 'goal_reached'
     });
-    await api.jobs.finish(gameId, jobId, {});
+    const closed = await api.jobs.finish(gameId, jobId, {});
+    if (closed && closed.refund_limited_note) log(closed.refund_limited_note);
     journal.finished = true;
     journal.runs[profileRun.run_id].completed = true;
     journal.runs[profileRun.run_id].state = 'succeeded';
@@ -201,9 +208,20 @@ async function profile(flags, injected, deps, log) {
   } catch (error) {
     heartbeat.stop();
     const code = exitCodeFor(error);
-    await safeFail({ api, gameId, jobId, runId: profileRun.run_id, error, log });
     log(`profile failed: ${String(error && error.message)}`);
-    return code;
+    if (Array.isArray(error && error.rawAnswers) && error.rawAnswers.length > 0) {
+      try {
+        const saved = await writeProfileFailure(directory, error.rawAnswers, String(error.message));
+        log(`The model's answers, secrets redacted, are in ${saved}.`);
+      } catch (writeError) {
+        log(`the model's answers could not be saved: ${String(writeError && writeError.message)}`);
+      }
+    }
+    const settled = await safeFail({ api, gameId, jobId, runId: profileRun.run_id, error, log });
+    if (settled) log(`job ${jobId} ${settled.job ? settled.job.state : 'failed'}: refunded ${formatCents(settled.refunded_cents)}.`);
+    if (settled && settled.refund_limited_note) log(settled.refund_limited_note);
+    // A failure is never exit 0, whatever the settlement says.
+    return code === EXIT.OK ? EXIT.FAILED : code;
   }
 }
 
@@ -223,9 +241,10 @@ async function safeFail({ api, gameId, jobId, runId, error, log }) {
     log(`the run could not be marked failed: ${String(completeError && completeError.message)}`);
   }
   try {
-    await api.jobs.finish(gameId, jobId, { reason: 'profile failed' });
+    return await api.jobs.finish(gameId, jobId, { reason: 'profile failed' });
   } catch (finishError) {
-    log(`the job could not be settled now; the hourly sweep will settle it: ${String(finishError && finishError.message)}`);
+    log(settleFailureLine(jobId, finishError));
+    return null;
   }
 }
 

package/src/commands/punch.js

@@ -0,0 +1,116 @@
+import { ui } from '../ui/index.js';
+import { ExitCode } from '../errors.js';
+import { guarded, printHints, printList, requireItemId, requireStatus, requireText, sourceWords, withList } from './lists.js';
+
+/**
+ * `ravensight-playtest punch`: the punch list (Labs beta).
+ *
+ * What has to be finished before handover, closed one item at a time. Reading,
+ * adding and closing are free. `draft` asks for proposals from the game's
+ * findings and specs, which is one AI unit, and saves nothing unless
+ * `--accept` is passed. Nothing here closes an item on its own: `fix li_...`
+ * records the fix job on the item and leaves closing it to a person.
+ */
+
+const LIST = 'punch';
+
+/** `punch list [--status open|done|dropped|all]` */
+export async function punchList(flags = {}, deps = {}) {
+  const status = requireStatus(flags.status);
+  return withList(flags, deps, async ({ api, gameId }) => {
+    const answer = await guarded(() => api.labs.lists.list(gameId, LIST, { status }), { list: LIST });
+    if (flags.json) {
+      ui.json(answer);
+      return ExitCode.OK;
+    }
+    printList(answer, LIST, status);
+    return ExitCode.OK;
+  });
+}
+
+/** `punch add "<text>"` */
+export async function punchAdd(text, flags = {}, deps = {}) {
+  const value = requireText(text);
+  return withList(flags, deps, async ({ api, gameId }) => {
+    const answer = await guarded(() => api.labs.lists.add(gameId, LIST, { text: value }), { list: LIST });
+    if (flags.json) {
+      ui.json(answer);
+      return ExitCode.OK;
+    }
+    printAdded(answer);
+    return ExitCode.OK;
+  });
+}
+
+/** `punch done <li_id>` and `punch drop <li_id>` */
+export async function punchClose(itemId, status, flags = {}, deps = {}) {
+  const id = requireItemId(itemId);
+  return withList(flags, deps, async ({ api, gameId }) => {
+    const answer = await guarded(() => api.labs.lists.update(gameId, LIST, id, { status }), { list: LIST, itemId: id });
+    if (flags.json) {
+      ui.json(answer);
+      return ExitCode.OK;
+    }
+    const text = answer && answer.item && answer.item.text ? `: ${answer.item.text}` : '';
+    ui.ok(`${status === 'done' ? 'Done' : 'Dropped'} ${id}${text}`);
+    return ExitCode.OK;
+  });
+}
+
+/** `punch draft [--accept]` */
+export async function punchDraft(flags = {}, deps = {}) {
+  return withList(flags, deps, async ({ api, gameId }) => {
+    if (!flags.json) {
+      ui.info('Drafting a punch list is one AI unit. It is not charged when there is nothing to propose.');
+      ui.info(flags.accept ? 'Every proposal will be added to the punch list.' : 'Nothing is saved. Pass --accept to add the proposals.');
+    }
+    const confirm = deps.confirm || ui.confirm;
+    if (!flags.yes && !(await confirm('Draft the punch list?', { yes: flags.yes }))) {
+      ui.info('Nothing was charged. Pass --yes to draft without a terminal.');
+      return ExitCode.CANCELED;
+    }
+    const answer = await guarded(() => api.labs.lists.draft(gameId), { list: LIST });
+    const proposals = answer && Array.isArray(answer.proposals) ? answer.proposals : [];
+    const added = [];
+    if (flags.accept) {
+      try {
+        for (const proposal of proposals) {
+          const body = proposal.source ? { text: proposal.text, source: proposal.source } : { text: proposal.text };
+          added.push(await guarded(() => api.labs.lists.add(gameId, LIST, body), { list: LIST }));
+        }
+      } catch (error) {
+        // Say what was saved before the refusal, so nobody adds it twice.
+        if (!flags.json) for (const saved of added) printAdded(saved);
+        throw error;
+      }
+    }
+    if (flags.json) {
+      ui.json({ ...answer, ...(flags.accept ? { added } : {}) });
+      return ExitCode.OK;
+    }
+    if (proposals.length === 0) {
+      ui.info('No proposals, so nothing was charged.');
+      return ExitCode.OK;
+    }
+    const units = Number(answer.units) || 0;
+    ui.info(`${proposals.length} proposal${proposals.length === 1 ? '' : 's'}, ${units} AI unit${units === 1 ? '' : 's'} charged:`);
+    if (flags.accept) {
+      for (const saved of added) printAdded(saved);
+      return ExitCode.OK;
+    }
+    proposals.forEach((proposal, index) => {
+      ui.info(`  ${index + 1}. ${proposal.text} (${sourceWords(proposal.source)})`);
+    });
+    ui.blank();
+    ui.info('Nothing was saved. Add the ones you want with "ravensight-playtest punch add".');
+    return ExitCode.OK;
+  });
+}
+
+/** One added item, with its plain-language hints under it. */
+export function printAdded(answer) {
+  const item = answer && answer.item ? answer.item : {};
+  if (answer && answer.existing) ui.info(`Already on the list: ${item.item_id}, ${item.text}`);
+  else ui.ok(`Added ${item.item_id}: ${item.text}`);
+  if (printHints(item) > 0) ui.info('  These hints follow the rules of Simplified Technical English. They never block an item.');
+}

package/src/commands/replay.js

@@ -0,0 +1,204 @@
+import { readdir, readFile } from 'node:fs/promises';
+import path from 'node:path';
+
+import { getDeps } from '../run/deps.js';
+import EXIT from '../run/exit.js';
+import { jobsDir, runDir as runDirFor } from '../run/paths.js';
+import { checkoutRef, replayTrace } from '../run/replay.js';
+import { findTraceIn, parseTrace, TRACE_FILE } from '../run/trace.js';
+
+/**
+ * `ravensight-playtest replay <runId> --ref <branch>`: play a recorded persona
+ * run again with no model and report the first place it stops matching.
+ *
+ * Free and local. The commit is extracted to a temp directory, so the developer's
+ * checkout is not touched. When this machine is signed in and the run's job is
+ * known, the outcome is recorded on the job, which is where the per-job cap
+ * lives; a replay that cannot be recorded still happened and still answers.
+ *
+ * WHAT IT PROVES. Matching means this seed and these inputs produced the same
+ * recorded state at every step, on this machine, on this Godot. It does not prove
+ * the game is deterministic in general. A divergence on a changed branch is
+ * evidence about the change only if the trace also matches where it was recorded,
+ * which is what `--baseline` checks and what the output says when it was not.
+ */
+
+/** The largest game-state value kept in a recorded divergence. */
+const MAX_DIFF_VALUE_CHARS = 200;
+
+/** Where a run's trace lives on this machine, by run id alone. */
+export async function findLocalTrace(repoRoot, runId, jobId = null) {
+  if (jobId) return { file: await findTraceIn(runDirFor(repoRoot, jobId, runId)), jobId };
+  const jobs = await readdir(jobsDir(repoRoot)).catch(() => []);
+  for (const candidate of jobs) {
+    const file = await findTraceIn(runDirFor(repoRoot, candidate, runId));
+    if (file) return { file, jobId: candidate };
+  }
+  return { file: null, jobId: null };
+}
+
+/**
+ * A run's trace text: this machine's copy, else the uploaded one. Null when
+ * neither exists, and never throws for an ordinary "not there".
+ *
+ * @returns {Promise<{text: string, jobId: string|null, source: string}|null>}
+ */
+export async function loadTrace({ repoRoot, runId, jobId = null, api = null, gameId = null, fetchImpl = fetch }) {
+  const local = await findLocalTrace(repoRoot, runId, jobId);
+  if (local.file) return { text: await readFile(local.file, 'utf8'), jobId: local.jobId, source: 'local' };
+  if (!api || !gameId || !jobId) return null;
+  try {
+    const { artifacts = [] } = await api.jobs.artifacts(gameId, jobId);
+    const entry = artifacts.find(item => item.run_id === runId && item.path === TRACE_FILE && item.status === 'verified');
+    if (!entry) return null;
+    const response = await fetchImpl(entry.url);
+    if (!response.ok) return null;
+    return { text: await response.text(), jobId, source: 'uploaded' };
+  } catch {
+    return null;
+  }
+}
+
+function boundedValue(value) {
+  const text = JSON.stringify(value ?? null);
+  return text.length <= MAX_DIFF_VALUE_CHARS ? value ?? null : `${text.slice(0, MAX_DIFF_VALUE_CHARS)}...`;
+}
+
+/** A replay result in the shape the server records. Hashes and differences, no state dump. */
+export function replayReport(result, { ref, sha }) {
+  const divergence = result.first_divergence;
+  return {
+    ref: String(ref).slice(0, 200),
+    sha,
+    matched: result.matched,
+    fully_replayed: result.fully_replayed,
+    steps_total: result.steps_total,
+    steps_compared: result.steps_compared,
+    ticks: result.ticks,
+    stopped: result.stopped || null,
+    first_divergence: divergence ? {
+      step: divergence.step,
+      ...(Number.isInteger(divergence.frame) ? { frame: divergence.frame } : {}),
+      ...(divergence.reason ? { reason: String(divergence.reason).slice(0, 500) } : {}),
+      ...(divergence.actual_error ? { actual_error: String(divergence.actual_error).slice(0, 500) } : {}),
+      ...(divergence.expected ? { expected_hash: divergence.expected.state_hash } : {}),
+      ...(divergence.actual ? { actual_hash: divergence.actual.state_hash } : {}),
+      ...(divergence.differing_subtrees ? { differing_subtrees: divergence.differing_subtrees.slice(0, 20).map(name => String(name).slice(0, 200)) } : {}),
+      ...(divergence.game_state_diff ? {
+        game_state_diff: Object.fromEntries(Object.entries(divergence.game_state_diff).slice(0, 20).map(([key, pair]) => [
+          key.slice(0, 80),
+          { expected: boundedValue(pair.expected), actual: boundedValue(pair.actual) }
+        ]))
+      } : {})
+    } : null
+  };
+}
+
+/** One sentence per outcome, in the same words `simulation-replay` uses. */
+export function replaySentence(result) {
+  if (result.matched) {
+    if (result.fully_replayed && (!(result.steps_compared > 0) || !(result.ticks > 0))) return 'Replay compared nothing: the recorded run took no step the game could be held to, so this says nothing about the game.';
+    if (result.fully_replayed) return `Replay matched every recorded state (${result.steps_compared} compared, ${result.ticks} ticks).`;
+    if (result.stopped === 'redacted_input') return 'Replay matched the recorded prefix only: the run typed a secret from the local brief here, and the trace does not hold it.';
+    return 'Replay matched the recorded prefix only. The trace does not establish a complete playthrough.';
+  }
+  const divergence = result.first_divergence || {};
+  const where = `step ${divergence.step}${Number.isInteger(divergence.frame) ? `, tick ${divergence.frame}` : ''}`;
+  if (divergence.reason) return `Replay diverged at ${where}: ${divergence.reason}${divergence.actual_error ? ` (${divergence.actual_error})` : ''}`;
+  if (divergence.actual_error) return `Replay diverged at ${where}: ${divergence.actual_error}`;
+  const diff = Object.entries(divergence.game_state_diff || {}).slice(0, 5)
+    .map(([key, pair]) => `${key} was ${JSON.stringify(pair.expected)}, now ${JSON.stringify(pair.actual)}`).join('; ');
+  const trees = (divergence.differing_subtrees || []).join(', ');
+  return `Replay diverged at ${where}.${diff ? ` ${diff}.` : ''}${trees ? ` Differs under: ${trees}.` : ''}`;
+}
+
+/**
+ * @param {string} runId
+ * @param {Object} flags
+ * @param {Object} [injected] test seam: deps, replayTrace, checkoutRef, log, fetch
+ * @returns {Promise<number>} an exit code
+ */
+export async function replayCommand(runId, flags = {}, injected = {}) {
+  const deps = injected.deps || (await getDeps());
+  const log = injected.log || (deps.ui && deps.ui.info ? message => deps.ui.info(message) : message => console.log(message));
+  const play = injected.replayTrace || replayTrace;
+  const checkout = injected.checkoutRef || checkoutRef;
+  const repoRoot = path.resolve(flags.repoRoot || process.cwd());
+
+  let context = null;
+  try {
+    context = await deps.createContext({ apiUrl: flags.apiUrl, repoRoot, requireAuth: false });
+  } catch {
+    context = null;
+  }
+  const api = context && context.token ? context.api : null;
+  const gameId = flags.game || (context && context.config && context.config.game_id) || null;
+
+  const trace = await loadTrace({ repoRoot, runId, jobId: flags.job || null, api, gameId, fetchImpl: injected.fetch || fetch });
+  if (!trace) {
+    log(`There is no trace for ${runId} on this machine${flags.job ? ' or on the server' : ''}. Pass --job <jobId> to read an uploaded one. A run records a trace only on the Godot driver, in lockstep.`);
+    return EXIT.ENVIRONMENT;
+  }
+  let parsed;
+  try {
+    parsed = parseTrace(trace.text);
+  } catch (error) {
+    log(String(error.message));
+    return EXIT.ENVIRONMENT;
+  }
+
+  const project = path.resolve(repoRoot, flags.godotProject || (context && context.config && context.config.godot_project) || '.');
+  const subdir = path.relative(repoRoot, project);
+  if (subdir.startsWith('..')) {
+    log('The Godot project has to be inside this repository to replay a commit of it.');
+    return EXIT.ENVIRONMENT;
+  }
+
+  const replayOn = async ref => {
+    const checked = await checkout({ repoRoot, ref, projectSubdir: subdir, godotPath: flags.godotPath });
+    try {
+      const result = await play({ text: trace.text, projectDir: checked.projectDir, godotPath: flags.godotPath, headless: flags.headless ? true : undefined });
+      return { result, sha: checked.sha };
+    } finally {
+      await checked.cleanup();
+    }
+  };
+
+  const ref = flags.ref || 'HEAD';
+  let baseline = null;
+  try {
+    if (flags.baseline) {
+      if (!parsed.header.commit) {
+        log('This trace does not say which commit it was recorded on, so there is nothing to check it against.');
+      } else {
+        baseline = await replayOn(parsed.header.commit);
+        log(`On the commit it was recorded on (${parsed.header.commit.slice(0, 12)}): ${replaySentence(baseline.result)}`);
+      }
+    }
+    const { result, sha } = await replayOn(ref);
+
+    let recorded = null;
+    if (api && gameId && trace.jobId && flags.record !== false) {
+      try {
+        recorded = await api.runs.recordReplay(gameId, trace.jobId, runId, replayReport(result, { ref, sha }));
+      } catch (error) {
+        log(`The replay was not recorded on the job: ${String(error && error.message)}`);
+      }
+    }
+
+    if (flags.json) {
+      (deps.ui && deps.ui.json ? deps.ui.json : value => console.log(JSON.stringify(value)))({ run_id: runId, ref, sha, ...result, baseline: baseline ? baseline.result : null, recorded });
+    } else {
+      log(`${ref} (${sha.slice(0, 12)}): ${replaySentence(result)}`);
+      if (!result.matched) {
+        if (baseline && !baseline.result.matched) log('The trace does not replay on the commit it was recorded on either, so this game is not reproducible on this seed and this divergence says nothing about the change.');
+        else if (baseline) log('It matches where it was recorded, so the change is what moved it.');
+        else log('Pass --baseline to check the trace against the commit it was recorded on. A game that does not honor its seed diverges everywhere, whatever changed.');
+      }
+    }
+    return result.matched ? EXIT.OK : EXIT.FAILED;
+  } catch (error) {
+    log(String(error && error.message));
+    return EXIT.ENVIRONMENT;
+  }
+}

package/src/commands/resume.js

@@ -85,6 +85,13 @@ async function resumeWith(context, jobId, flags, deps) {
       { hint: 'Run resume with no job id to see what is here.' }
     );
   }
+  if (state.options && state.options.labs === 'feature') {
+    throw new CliError(
+      `${jobId} is a Build a feature job, and those cannot be resumed.`,
+      1,
+      { hint: 'The hourly sweep settles it and gives back what did not happen. The branch keeps whatever was written.' }
+    );
+  }
   const gameId = flags.game || state.game_id;
   if (!gameId) throw new CliError(`The journal for ${jobId} does not say which game it belongs to.`);