ravensight-playtest 0.5.2 → 0.6.0

package/src/commands/init.js

@@ -6,6 +6,8 @@ import { withAuthHandling } from '../auth/session.js';
 import { paths } from '../paths.js';
 import { ui } from '../ui/index.js';
 import { CliError, ExitCode } from '../errors.js';
+import { recommendConcurrency } from '../detect.js';
+import { isGodotProject } from '../run/drivers/godot-project.js';
 
 const GITIGNORE_LINE = '.ravensight/jobs/';
 
@@ -66,7 +68,7 @@ export async function init(flags = {}, deps = {}) {
   // Every API call in this command goes through the one 401 handler, so a
   // refused credential is cleared and explained here the same way it is
   // everywhere else.
-  return withAuthHandling(context, () => linkRepo(context, flags, repoRoot, deps.choose || ui.choose));
+  return withAuthHandling(context, () => linkRepo(context, flags, repoRoot, deps.choose || ui.choose, deps.recommend || recommendConcurrency));
 }
 
 /**
@@ -76,7 +78,7 @@ export async function init(flags = {}, deps = {}) {
  * @param {(question: string, labels: Array<string>) => Promise<number|null>} choose
  *   the picker; answers null when there is no terminal to ask on
  */
-async function linkRepo(context, flags, repoRoot, choose) {
+async function linkRepo(context, flags, repoRoot, choose, recommend) {
   const whoami = await context.api.cli.whoami();
 
   const games = whoami.games || [];
@@ -130,8 +132,20 @@ async function linkRepo(context, flags, repoRoot, choose) {
     );
   }
 
+  // What check would say: a repository with a project.godot is played with
+  // the Godot driver, and the concurrency is the one this machine can take.
+  // 0.5.0 wrote playwright_web and 2 right after check said Godot and 1.
+  const godot = Boolean(await isGodotProject(repoRoot));
+  const machine = recommend();
   const config = await saveConfig(
-    { ...(existing || {}), ...defaultConfig(gameId), api_url: context.apiUrl },
+    {
+      ...(existing || {}),
+      ...defaultConfig(gameId, {
+        concurrency: machine.recommended,
+        ...(godot ? { driver: 'godot_driver', build: { kind: 'godot_project', ref: '.' } } : {})
+      }),
+      api_url: context.apiUrl
+    },
     { repoRoot }
   );
   const gitignore = await ensureGitignore(repoRoot);
@@ -144,6 +158,8 @@ async function linkRepo(context, flags, repoRoot, choose) {
   const name = (games.find(game => game.gameId === gameId) || {}).name || gameId;
   ui.ok(`Linked this repo to ${name} (${gameId}).`);
   ui.info(`Wrote ${paths.configFile({ repoRoot })}`);
+  if (godot) ui.info('This is a Godot project, so runs use godot_driver on this directory.');
+  ui.info(`Concurrency ${config.concurrency}, from this machine (${machine.cpus} logical cpus, ${machine.freeMemGb} GB free).`);
   if (gitignore === 'added') ui.info(`Added ${GITIGNORE_LINES.join(' and ')} to .gitignore`);
   if (gitignore === 'absent') ui.warn(`No .gitignore here. Add ${GITIGNORE_LINES.join(' and ')} to whatever ignores build output.`);
   ui.blank();

package/src/commands/profile.js

@@ -1,12 +1,13 @@
 import { randomUUID } from 'node:crypto';
 import { mkdir } from 'node:fs/promises';
 import { optionsFrom, unknownOptionNotes } from '../run/args.js';
+import { settleFailureLine } from '../run/settle.js';
 import { getDeps } from '../run/deps.js';
 import EXIT from '../run/exit.js';
 import { createHeartbeat } from '../run/heartbeat.js';
 import { createModelClient, ModelUnavailableError, ProxyError } from '../run/model.js';
 import { jobDir } from '../run/paths.js';
-import { runGameProfile, writeCapabilityReport } from '../run/profile.js';
+import { runGameProfile, writeCapabilityReport, writeProfileFailure } from '../run/profile.js';
 import * as stateFile from '../run/state.js';
 import { createUsageLedger, writeUsageFile } from '../run/usage.js';
 import { formatCents, resolveSetup } from './run.js';
@@ -64,6 +65,11 @@ async function profile(flags, injected, deps, log) {
   const { context, api, gameId, apiUrl, repoRoot, cliVersion } = setup;
 
   const modules = ['game_profile'];
+  // The profile is a read of the repository: --driver, --godot-project and
+  // --godot-path are accepted for symmetry with run and change nothing here.
+  if (options.driver || options.godotProject || options.godotPath) {
+    log('profile reads the repository and does not launch the game, so --driver, --godot-project and --godot-path are not used here. A persona run (run) is what plays the game.');
+  }
   const pack = await deps.packs.get({ api, gameId, modules });
   // A profile job is the one shape that does not need a brief, so a missing one
   // is not an error here. It is still read when there is one: a brief that
@@ -186,7 +192,8 @@ async function profile(flags, injected, deps, log) {
       actions_taken: turns,
       quit_reason: 'goal_reached'
     });
-    await api.jobs.finish(gameId, jobId, {});
+    const closed = await api.jobs.finish(gameId, jobId, {});
+    if (closed && closed.refund_limited_note) log(closed.refund_limited_note);
     journal.finished = true;
     journal.runs[profileRun.run_id].completed = true;
     journal.runs[profileRun.run_id].state = 'succeeded';
@@ -201,9 +208,20 @@ async function profile(flags, injected, deps, log) {
   } catch (error) {
     heartbeat.stop();
     const code = exitCodeFor(error);
-    await safeFail({ api, gameId, jobId, runId: profileRun.run_id, error, log });
     log(`profile failed: ${String(error && error.message)}`);
-    return code;
+    if (Array.isArray(error && error.rawAnswers) && error.rawAnswers.length > 0) {
+      try {
+        const saved = await writeProfileFailure(directory, error.rawAnswers, String(error.message));
+        log(`The model's answers, secrets redacted, are in ${saved}.`);
+      } catch (writeError) {
+        log(`the model's answers could not be saved: ${String(writeError && writeError.message)}`);
+      }
+    }
+    const settled = await safeFail({ api, gameId, jobId, runId: profileRun.run_id, error, log });
+    if (settled) log(`job ${jobId} ${settled.job ? settled.job.state : 'failed'}: refunded ${formatCents(settled.refunded_cents)}.`);
+    if (settled && settled.refund_limited_note) log(settled.refund_limited_note);
+    // A failure is never exit 0, whatever the settlement says.
+    return code === EXIT.OK ? EXIT.FAILED : code;
   }
 }
 
@@ -223,9 +241,10 @@ async function safeFail({ api, gameId, jobId, runId, error, log }) {
     log(`the run could not be marked failed: ${String(completeError && completeError.message)}`);
   }
   try {
-    await api.jobs.finish(gameId, jobId, { reason: 'profile failed' });
+    return await api.jobs.finish(gameId, jobId, { reason: 'profile failed' });
   } catch (finishError) {
-    log(`the job could not be settled now; the hourly sweep will settle it: ${String(finishError && finishError.message)}`);
+    log(settleFailureLine(jobId, finishError));
+    return null;
   }
 }
 

package/src/commands/replay.js

@@ -0,0 +1,204 @@
+import { readdir, readFile } from 'node:fs/promises';
+import path from 'node:path';
+
+import { getDeps } from '../run/deps.js';
+import EXIT from '../run/exit.js';
+import { jobsDir, runDir as runDirFor } from '../run/paths.js';
+import { checkoutRef, replayTrace } from '../run/replay.js';
+import { findTraceIn, parseTrace, TRACE_FILE } from '../run/trace.js';
+
+/**
+ * `ravensight-playtest replay <runId> --ref <branch>`: play a recorded persona
+ * run again with no model and report the first place it stops matching.
+ *
+ * Free and local. The commit is extracted to a temp directory, so the developer's
+ * checkout is not touched. When this machine is signed in and the run's job is
+ * known, the outcome is recorded on the job, which is where the per-job cap
+ * lives; a replay that cannot be recorded still happened and still answers.
+ *
+ * WHAT IT PROVES. Matching means this seed and these inputs produced the same
+ * recorded state at every step, on this machine, on this Godot. It does not prove
+ * the game is deterministic in general. A divergence on a changed branch is
+ * evidence about the change only if the trace also matches where it was recorded,
+ * which is what `--baseline` checks and what the output says when it was not.
+ */
+
+/** The largest game-state value kept in a recorded divergence. */
+const MAX_DIFF_VALUE_CHARS = 200;
+
+/** Where a run's trace lives on this machine, by run id alone. */
+export async function findLocalTrace(repoRoot, runId, jobId = null) {
+  if (jobId) return { file: await findTraceIn(runDirFor(repoRoot, jobId, runId)), jobId };
+  const jobs = await readdir(jobsDir(repoRoot)).catch(() => []);
+  for (const candidate of jobs) {
+    const file = await findTraceIn(runDirFor(repoRoot, candidate, runId));
+    if (file) return { file, jobId: candidate };
+  }
+  return { file: null, jobId: null };
+}
+
+/**
+ * A run's trace text: this machine's copy, else the uploaded one. Null when
+ * neither exists, and never throws for an ordinary "not there".
+ *
+ * @returns {Promise<{text: string, jobId: string|null, source: string}|null>}
+ */
+export async function loadTrace({ repoRoot, runId, jobId = null, api = null, gameId = null, fetchImpl = fetch }) {
+  const local = await findLocalTrace(repoRoot, runId, jobId);
+  if (local.file) return { text: await readFile(local.file, 'utf8'), jobId: local.jobId, source: 'local' };
+  if (!api || !gameId || !jobId) return null;
+  try {
+    const { artifacts = [] } = await api.jobs.artifacts(gameId, jobId);
+    const entry = artifacts.find(item => item.run_id === runId && item.path === TRACE_FILE && item.status === 'verified');
+    if (!entry) return null;
+    const response = await fetchImpl(entry.url);
+    if (!response.ok) return null;
+    return { text: await response.text(), jobId, source: 'uploaded' };
+  } catch {
+    return null;
+  }
+}
+
+function boundedValue(value) {
+  const text = JSON.stringify(value ?? null);
+  return text.length <= MAX_DIFF_VALUE_CHARS ? value ?? null : `${text.slice(0, MAX_DIFF_VALUE_CHARS)}...`;
+}
+
+/** A replay result in the shape the server records. Hashes and differences, no state dump. */
+export function replayReport(result, { ref, sha }) {
+  const divergence = result.first_divergence;
+  return {
+    ref: String(ref).slice(0, 200),
+    sha,
+    matched: result.matched,
+    fully_replayed: result.fully_replayed,
+    steps_total: result.steps_total,
+    steps_compared: result.steps_compared,
+    ticks: result.ticks,
+    stopped: result.stopped || null,
+    first_divergence: divergence ? {
+      step: divergence.step,
+      ...(Number.isInteger(divergence.frame) ? { frame: divergence.frame } : {}),
+      ...(divergence.reason ? { reason: String(divergence.reason).slice(0, 500) } : {}),
+      ...(divergence.actual_error ? { actual_error: String(divergence.actual_error).slice(0, 500) } : {}),
+      ...(divergence.expected ? { expected_hash: divergence.expected.state_hash } : {}),
+      ...(divergence.actual ? { actual_hash: divergence.actual.state_hash } : {}),
+      ...(divergence.differing_subtrees ? { differing_subtrees: divergence.differing_subtrees.slice(0, 20).map(name => String(name).slice(0, 200)) } : {}),
+      ...(divergence.game_state_diff ? {
+        game_state_diff: Object.fromEntries(Object.entries(divergence.game_state_diff).slice(0, 20).map(([key, pair]) => [
+          key.slice(0, 80),
+          { expected: boundedValue(pair.expected), actual: boundedValue(pair.actual) }
+        ]))
+      } : {})
+    } : null
+  };
+}
+
+/** One sentence per outcome, in the same words `simulation-replay` uses. */
+export function replaySentence(result) {
+  if (result.matched) {
+    if (result.fully_replayed && (!(result.steps_compared > 0) || !(result.ticks > 0))) return 'Replay compared nothing: the recorded run took no step the game could be held to, so this says nothing about the game.';
+    if (result.fully_replayed) return `Replay matched every recorded state (${result.steps_compared} compared, ${result.ticks} ticks).`;
+    if (result.stopped === 'redacted_input') return 'Replay matched the recorded prefix only: the run typed a secret from the local brief here, and the trace does not hold it.';
+    return 'Replay matched the recorded prefix only. The trace does not establish a complete playthrough.';
+  }
+  const divergence = result.first_divergence || {};
+  const where = `step ${divergence.step}${Number.isInteger(divergence.frame) ? `, tick ${divergence.frame}` : ''}`;
+  if (divergence.reason) return `Replay diverged at ${where}: ${divergence.reason}${divergence.actual_error ? ` (${divergence.actual_error})` : ''}`;
+  if (divergence.actual_error) return `Replay diverged at ${where}: ${divergence.actual_error}`;
+  const diff = Object.entries(divergence.game_state_diff || {}).slice(0, 5)
+    .map(([key, pair]) => `${key} was ${JSON.stringify(pair.expected)}, now ${JSON.stringify(pair.actual)}`).join('; ');
+  const trees = (divergence.differing_subtrees || []).join(', ');
+  return `Replay diverged at ${where}.${diff ? ` ${diff}.` : ''}${trees ? ` Differs under: ${trees}.` : ''}`;
+}
+
+/**
+ * @param {string} runId
+ * @param {Object} flags
+ * @param {Object} [injected] test seam: deps, replayTrace, checkoutRef, log, fetch
+ * @returns {Promise<number>} an exit code
+ */
+export async function replayCommand(runId, flags = {}, injected = {}) {
+  const deps = injected.deps || (await getDeps());
+  const log = injected.log || (deps.ui && deps.ui.info ? message => deps.ui.info(message) : message => console.log(message));
+  const play = injected.replayTrace || replayTrace;
+  const checkout = injected.checkoutRef || checkoutRef;
+  const repoRoot = path.resolve(flags.repoRoot || process.cwd());
+
+  let context = null;
+  try {
+    context = await deps.createContext({ apiUrl: flags.apiUrl, repoRoot, requireAuth: false });
+  } catch {
+    context = null;
+  }
+  const api = context && context.token ? context.api : null;
+  const gameId = flags.game || (context && context.config && context.config.game_id) || null;
+
+  const trace = await loadTrace({ repoRoot, runId, jobId: flags.job || null, api, gameId, fetchImpl: injected.fetch || fetch });
+  if (!trace) {
+    log(`There is no trace for ${runId} on this machine${flags.job ? ' or on the server' : ''}. Pass --job <jobId> to read an uploaded one. A run records a trace only on the Godot driver, in lockstep.`);
+    return EXIT.ENVIRONMENT;
+  }
+  let parsed;
+  try {
+    parsed = parseTrace(trace.text);
+  } catch (error) {
+    log(String(error.message));
+    return EXIT.ENVIRONMENT;
+  }
+
+  const project = path.resolve(repoRoot, flags.godotProject || (context && context.config && context.config.godot_project) || '.');
+  const subdir = path.relative(repoRoot, project);
+  if (subdir.startsWith('..')) {
+    log('The Godot project has to be inside this repository to replay a commit of it.');
+    return EXIT.ENVIRONMENT;
+  }
+
+  const replayOn = async ref => {
+    const checked = await checkout({ repoRoot, ref, projectSubdir: subdir, godotPath: flags.godotPath });
+    try {
+      const result = await play({ text: trace.text, projectDir: checked.projectDir, godotPath: flags.godotPath, headless: flags.headless ? true : undefined });
+      return { result, sha: checked.sha };
+    } finally {
+      await checked.cleanup();
+    }
+  };
+
+  const ref = flags.ref || 'HEAD';
+  let baseline = null;
+  try {
+    if (flags.baseline) {
+      if (!parsed.header.commit) {
+        log('This trace does not say which commit it was recorded on, so there is nothing to check it against.');
+      } else {
+        baseline = await replayOn(parsed.header.commit);
+        log(`On the commit it was recorded on (${parsed.header.commit.slice(0, 12)}): ${replaySentence(baseline.result)}`);
+      }
+    }
+    const { result, sha } = await replayOn(ref);
+
+    let recorded = null;
+    if (api && gameId && trace.jobId && flags.record !== false) {
+      try {
+        recorded = await api.runs.recordReplay(gameId, trace.jobId, runId, replayReport(result, { ref, sha }));
+      } catch (error) {
+        log(`The replay was not recorded on the job: ${String(error && error.message)}`);
+      }
+    }
+
+    if (flags.json) {
+      (deps.ui && deps.ui.json ? deps.ui.json : value => console.log(JSON.stringify(value)))({ run_id: runId, ref, sha, ...result, baseline: baseline ? baseline.result : null, recorded });
+    } else {
+      log(`${ref} (${sha.slice(0, 12)}): ${replaySentence(result)}`);
+      if (!result.matched) {
+        if (baseline && !baseline.result.matched) log('The trace does not replay on the commit it was recorded on either, so this game is not reproducible on this seed and this divergence says nothing about the change.');
+        else if (baseline) log('It matches where it was recorded, so the change is what moved it.');
+        else log('Pass --baseline to check the trace against the commit it was recorded on. A game that does not honor its seed diverges everywhere, whatever changed.');
+      }
+    }
+    return result.matched ? EXIT.OK : EXIT.FAILED;
+  } catch (error) {
+    log(String(error && error.message));
+    return EXIT.ENVIRONMENT;
+  }
+}

package/src/commands/resume.js

@@ -85,6 +85,13 @@ async function resumeWith(context, jobId, flags, deps) {
       { hint: 'Run resume with no job id to see what is here.' }
     );
   }
+  if (state.options && state.options.labs === 'feature') {
+    throw new CliError(
+      `${jobId} is a Build a feature job, and those cannot be resumed.`,
+      1,
+      { hint: 'The hourly sweep settles it and gives back what did not happen. The branch keeps whatever was written.' }
+    );
+  }
   const gameId = flags.game || state.game_id;
   if (!gameId) throw new CliError(`The journal for ${jobId} does not say which game it belongs to.`);