ravensight-playtest 0.5.2 → 0.6.0

package/src/run/profile.js

@@ -6,7 +6,7 @@ import { callStepWithRetry, STEPS } from './model.js';
 import { JOB_FILES } from './paths.js';
 import { parseJsonObject } from './json.js';
 import { createRepoTools, PROFILE_TOOLS, READ_ONLY_TOOLS } from './repoTools.js';
-import { assertNoSecrets } from './secretScan.js';
+import { assertNoSecrets, redactSecrets } from './secretScan.js';
 
 /**
  * The `game_profile` module: read the developer's repository and answer what
@@ -38,6 +38,12 @@ export const MAX_TRIAGE_TURNS = 40;
 export const TRIAGE_MAX_TOKENS = 1024;
 export const SYNTHESIS_MAX_TOKENS = 4096;
 
+/** Files the synthesis step may still read before it has to answer. */
+export const MAX_SYNTHESIS_TOOL_TURNS = 3;
+
+/** Where a failed synthesis leaves its raw answers, in the job directory. */
+export const PROFILE_FAILURE_FILE = 'capability-report.failed.txt';
+
 /**
  * @param {Array<{name: string, content: Object}>} schemas
  */
@@ -159,7 +165,10 @@ export async function runGameProfile({
 
   let report = null;
   let lint = [];
-  for (let attempt = 1; attempt <= 2; attempt += 1) {
+  // Every synthesis answer, kept so a failed profile leaves something to read.
+  const rawAnswers = [];
+  let toolTurns = 0;
+  for (let attempt = 1; attempt <= 2;) {
     const answer = await callStepWithRetry({
       client,
       jobId,
@@ -171,12 +180,36 @@ export async function runGameProfile({
       maxTokens: SYNTHESIS_MAX_TOKENS
     });
     if (ledger) ledger.record({ model: answer.model, usage: answer.usage });
+    rawAnswers.push({ attempt, stopReason: answer.stopReason || null, text: answer.text || '', tools: answer.toolUses.map(toolUse => toolUse.name) });
+
+    // The step is given read_file, so it may use it. An answer that reads a
+    // file instead of writing the report used to be judged as a report with
+    // no JSON in it, twice. Read it, hand the file back, and ask again, a
+    // bounded number of times, without spending the repair.
+    if (answer.toolUses.length > 0 && toolTurns < MAX_SYNTHESIS_TOOL_TURNS && !parseJsonObject(answer.text)) {
+      toolTurns += 1;
+      messages.push({
+        role: 'assistant',
+        content: [
+          ...(answer.text.trim() ? [{ type: 'text', text: answer.text }] : []),
+          ...answer.toolUses.map(toolUse => ({ type: 'tool_use', id: toolUse.id, name: toolUse.name, input: toolUse.malformed ? {} : toolUse.input }))
+        ]
+      });
+      const results = [];
+      for (const toolUse of answer.toolUses) results.push((await repo.execute(toolUse)).result);
+      messages.push({
+        role: 'user',
+        content: [...results, { type: 'text', text: toolTurns >= MAX_SYNTHESIS_TOOL_TURNS ? 'That was the last file. Answer now with the capability report JSON object only.' : 'Answer with the capability report JSON object only when you have what you need.' }]
+      });
+      continue;
+    }
 
     const parsed = parseJsonObject(answer.text);
     const candidate = parsed ? { ...parsed, schema_version: parsed.schema_version || '1.0', job_id: jobId } : null;
+    const cutOff = answer.stopReason === 'max_tokens';
     const validation = candidate
       ? validator.validate(candidate)
-      : { ok: false, details: [{ instancePath: '/', message: 'the answer was not a JSON object' }] };
+      : { ok: false, details: [{ instancePath: '/', message: cutOff ? `the answer was cut off at the output limit (${SYNTHESIS_MAX_TOKENS} tokens) before the JSON object closed` : 'the answer was not a JSON object' }] };
 
     if (validation.ok) {
       report = candidate;
@@ -184,13 +217,20 @@ export async function runGameProfile({
       break;
     }
     lint = validation.details.map(detail => `${detail.instancePath}: ${detail.message}`);
-    if (attempt === 2) throw new Error(`the capability report did not validate after one repair: ${lint.join('; ')}`);
+    if (attempt === 2) {
+      const error = new Error(`the capability report did not validate after one repair: ${lint.join('; ')}`);
+      error.rawAnswers = rawAnswers;
+      throw error;
+    }
 
-    messages.push({ role: 'assistant', content: [{ type: 'text', text: answer.text.slice(0, 2000) }] });
+    messages.push({ role: 'assistant', content: [{ type: 'text', text: answer.text.slice(0, 2000) || '(no text)' }] });
     messages.push({
       role: 'user',
-      content: [{ type: 'text', text: `That did not match capability-report.schema.json. Fix exactly these and answer with the JSON only:\n${lint.join('\n')}` }]
+      content: [{ type: 'text', text: cutOff
+        ? `That answer was cut off at the output limit. Answer again with the JSON only, shorter: keep every required field and cut reasons and warnings to one sentence each.\n${lint.join('\n')}`
+        : `That did not match capability-report.schema.json. Fix exactly these and answer with the JSON only:\n${lint.join('\n')}` }]
     });
+    attempt += 1;
   }
 
   return { report, lint, turns };
@@ -209,6 +249,27 @@ export async function writeCapabilityReport(jobDirectory, report) {
   return target;
 }
 
+/**
+ * Write a failed synthesis's raw answers next to the job, secrets redacted,
+ * so a profile that was charged and refunded still says what went wrong.
+ *
+ * @param {string} jobDirectory
+ * @param {Array<{attempt: number, stopReason: string|null, text: string}>} rawAnswers
+ * @param {string} reason
+ * @returns {Promise<string>} the file written
+ */
+export async function writeProfileFailure(jobDirectory, rawAnswers, reason) {
+  const lines = [`profile failed: ${reason}`, ''];
+  (rawAnswers || []).forEach((answer, index) => {
+    lines.push(`--- synthesis answer ${index + 1} (attempt ${answer.attempt}, stop reason ${answer.stopReason || 'unknown'}${answer.tools && answer.tools.length > 0 ? `, tools ${answer.tools.join(', ')}` : ''}) ---`);
+    lines.push(answer.text || '(no text)');
+    lines.push('');
+  });
+  const target = path.join(jobDirectory, PROFILE_FAILURE_FILE);
+  await writeFile(target, redactSecrets(lines.join('\n')), { encoding: 'utf8', mode: 0o600 });
+  return target;
+}
+
 export { parseJsonObject };
 
-export default { runGameProfile, writeCapabilityReport, createCapabilityValidator, parseJsonObject };
+export default { runGameProfile, writeCapabilityReport, writeProfileFailure, createCapabilityValidator, parseJsonObject };

package/src/run/replay.js

@@ -0,0 +1,236 @@
+import { execFile, spawn } from 'node:child_process';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { promisify } from 'node:util';
+
+import { DriverRpcError } from './drivers/godot-rpc.js';
+import { allowlistedEnv, GodotDriver } from './drivers/godot.js';
+import { detectGodotBinary, importProjectCopy } from './drivers/godot-project.js';
+import { parseTrace } from './trace.js';
+
+const execFileAsync = promisify(execFile);
+
+/**
+ * Plays a recorded trace against a build, with no model anywhere in the path,
+ * and says where it first stopped matching.
+ *
+ * The answer has the same shape as `simulation-replay`, so one reader handles
+ * both: `matched`, `matched_prefix`, `fully_replayed`, `failure_reproduced`, and
+ * `first_divergence` as `{ step, expected, actual }`, `{ step, reason }` or
+ * `{ step, actual_error }`.
+ *
+ * A match says this seed and these inputs produced the same recorded state. A
+ * divergence on a changed branch is only evidence about the change if the trace
+ * also matches on the code it was recorded on, which is what `--baseline` checks
+ * and what the report says when it was not checked.
+ */
+
+/** The compact state a divergence names, in the simulation format's spelling. */
+function stateRecord(digest) {
+  if (!digest) return null;
+  return {
+    done: false,
+    state_hash: digest.state_hash,
+    scene: digest.scene,
+    nodes: digest.nodes,
+    ...(digest.game_state ? { game_state: digest.game_state } : {}),
+  };
+}
+
+function differingKeys(left = {}, right = {}, limit = 20) {
+  const keys = [...new Set([...Object.keys(left), ...Object.keys(right)])].sort();
+  return keys.filter((key) => left[key] !== right[key]).slice(0, limit);
+}
+
+/** Which keys of the game's own state hook differ, with both values. */
+function gameStateDiff(expected, actual) {
+  const out = {};
+  const hooks = new Set([...Object.keys(expected ?? {}), ...Object.keys(actual ?? {})]);
+  for (const hook of hooks) {
+    const left = expected?.[hook] ?? {};
+    const right = actual?.[hook] ?? {};
+    for (const key of new Set([...Object.keys(left), ...Object.keys(right)])) {
+      if (JSON.stringify(left[key]) !== JSON.stringify(right[key])) {
+        out[key] = { expected: left[key] ?? null, actual: right[key] ?? null };
+      }
+      if (Object.keys(out).length >= 20) return out;
+    }
+  }
+  return out;
+}
+
+function divergence(step, frame, expected, actual) {
+  const diff = gameStateDiff(expected?.game_state, actual?.game_state);
+  return {
+    step,
+    frame,
+    expected: stateRecord(expected),
+    actual: stateRecord(actual),
+    differing_subtrees: differingKeys(expected?.subtrees, actual?.subtrees),
+    ...(Object.keys(diff).length ? { game_state_diff: diff } : {}),
+  };
+}
+
+/**
+ * @param {Object} input
+ * @param {string} input.text - the trace file's contents
+ * @param {string} input.projectDir - the project to replay against
+ * @param {string} [input.godotPath]
+ * @param {boolean} [input.headless] - default the mode it was recorded in
+ * @param {string} [input.runDir] - where godot.log goes
+ * @param {AbortSignal} [input.signal]
+ * @param {(options: Object) => Object} [input.makeDriver] - test seam
+ * @returns {Promise<Object>} the simulation replay's result shape, plus counts
+ */
+export async function replayTrace({ text, projectDir, godotPath, headless, runDir, signal, env, makeDriver = options => new GodotDriver(options) }) {
+  const trace = parseTrace(text);
+  const { header, boot, steps } = trace;
+  const driver = makeDriver({
+    projectDir,
+    runDir,
+    godotPath,
+    headless: headless ?? Boolean(header.headless),
+    viewport: header.viewport || undefined,
+    lockstep: true,
+    seed: header.seed,
+    recordTrace: false,
+    ...(env ? { env } : {}),
+  });
+  const result = {
+    matched: true,
+    matched_prefix: false,
+    fully_replayed: false,
+    failure_reproduced: null,
+    first_divergence: null,
+    seed: header.seed,
+    steps_total: steps.length,
+    steps_compared: 0,
+    ticks: 0,
+    trace_truncated: trace.truncated,
+    stopped: null,
+  };
+  try {
+    await driver.launch({ signal });
+  } catch (error) {
+    result.matched = false;
+    result.first_divergence = { step: 0, frame: 0, actual_error: String(error?.message ?? error).slice(0, 500) };
+    result.boot_failed = true;
+    return result;
+  }
+  try {
+    if (driver.ticksPerSecond !== header.ticks_per_second) {
+      result.matched = false;
+      result.first_divergence = { step: 0, frame: 0, reason: `The project runs at ${driver.ticksPerSecond} ticks a second; this trace was recorded at ${header.ticks_per_second}.` };
+      return result;
+    }
+    if (boot && boot.state_hash !== driver.bootDigest?.state_hash) {
+      result.matched = false;
+      result.first_divergence = divergence('boot', 0, boot, driver.bootDigest);
+      return result;
+    }
+    let compared = 0;
+    for (const step of steps) {
+      if (signal?.aborted) throw new Error('Replay canceled.');
+      if (step.redacted) {
+        result.stopped = 'redacted_input';
+        break;
+      }
+      let answer;
+      try {
+        answer = await driver.step(step.events, step.ticks, { digest: Boolean(step.state) });
+      } catch (error) {
+        result.matched = false;
+        result.first_divergence = error instanceof DriverRpcError
+          ? { step: step.i, frame: step.frame, reason: 'Recorded action is no longer legal.', actual_error: error.rpcMessage }
+          : { step: step.i, frame: step.frame, actual_error: String(error?.message ?? error).slice(0, 500) };
+        break;
+      }
+      result.ticks = answer.tick;
+      if (answer.tick !== step.frame + step.ticks) {
+        result.matched = false;
+        result.first_divergence = { step: step.i, frame: step.frame, reason: `The game was at tick ${answer.tick} where the trace was at ${step.frame + step.ticks}.` };
+        break;
+      }
+      if (step.state) {
+        compared += 1;
+        if (step.state.state_hash !== answer.digest?.state_hash) {
+          result.matched = false;
+          result.first_divergence = divergence(step.i, step.frame, step.state, answer.digest);
+          break;
+        }
+      }
+    }
+    result.steps_compared = compared;
+    result.errors = driver.errorCount;
+    // A held key the desktop dropped and the driver put back: this replay may be a
+    // tick off for a reason that has nothing to do with the game.
+    result.inputs_reasserted = driver.reasserted ?? 0;
+    if (result.matched) {
+      result.matched_prefix = true;
+      result.fully_replayed = !trace.truncated && !result.stopped;
+    }
+    return result;
+  } finally {
+    await driver.stop({ reason: 'replay' }).catch(() => {});
+  }
+}
+
+/**
+ * The checkout's commit when its tree is clean, else null.
+ *
+ * @param {string} repoRoot
+ * @returns {Promise<string|null>}
+ */
+export async function cleanCommit(repoRoot) {
+  try {
+    const { stdout: status } = await execFileAsync('git', ['-C', repoRoot, 'status', '--porcelain', '--', '.', ':(exclude).ravensight']);
+    if (status.trim()) return null;
+    const { stdout } = await execFileAsync('git', ['-C', repoRoot, 'rev-parse', 'HEAD']);
+    return stdout.trim();
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Extract one commit to a temp directory without touching the developer's
+ * checkout, index or worktree list: `git archive`, so nothing is written to
+ * their repository at all. Untracked and ignored files are not in it, which
+ * includes `.godot/` (imported here, once) and any ignored key file.
+ *
+ * @param {{repoRoot: string, ref: string, projectSubdir?: string, godotPath?: string, importProject?: Function, env?: Object}} input
+ * @returns {Promise<{sha: string, ref: string, root: string, projectDir: string, imported: Object, cleanup: () => Promise<void>}>}
+ */
+export async function checkoutRef({ repoRoot, ref, projectSubdir = '', godotPath, importProject = importProjectCopy, env = process.env }) {
+  let sha;
+  try {
+    ({ stdout: sha } = await execFileAsync('git', ['-C', repoRoot, 'rev-parse', '--verify', `${ref}^{commit}`]));
+  } catch {
+    throw new Error(`There is no commit called "${ref}" in this repository.`);
+  }
+  sha = sha.trim();
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ravensight-ref-'));
+  const cleanup = () => fs.rm(root, { recursive: true, force: true }).catch(() => {});
+  try {
+    await new Promise((resolve, reject) => {
+      const archive = spawn('git', ['-C', repoRoot, 'archive', '--format=tar', sha], { stdio: ['ignore', 'pipe', 'ignore'] });
+      const tar = spawn('tar', ['-x', '-C', root], { stdio: ['pipe', 'ignore', 'ignore'] });
+      archive.stdout.pipe(tar.stdin);
+      archive.on('error', reject);
+      tar.on('error', reject);
+      tar.on('exit', code => (code === 0 ? resolve() : reject(new Error(`Extracting ${ref} failed.`))));
+    });
+    const projectDir = path.join(root, projectSubdir);
+    await fs.access(path.join(projectDir, 'project.godot')).catch(() => {
+      throw new Error(`${ref} has no Godot project at ${projectSubdir || 'the repository root'}.`);
+    });
+    const godot = await detectGodotBinary({ explicitPath: godotPath, env, projectDir });
+    // A branch's editor plugins run during this import. They get what a game gets.
+    const imported = await importProject({ godotPath: godot.path, projectCopy: projectDir, env: allowlistedEnv(env), logPath: null });
+    return { sha, ref, root, projectDir, imported, cleanup };
+  } catch (error) {
+    await cleanup();
+    throw error;
+  }
+}