ravensight-playtest 0.3.1 → 0.4.0

package/src/run/report.js

@@ -207,6 +207,191 @@ export function normaliseEvidence(report, runDirectory = null) {
   return { report: { ...report, findings }, changed };
 }
 
+/** Category words a model writes that are not in the schema, mapped to one that is, when the schema has it. */
+export const CATEGORY_SYNONYMS = Object.freeze({
+  ui: 'ux', usability: 'ux', onboarding: 'ux', clarity: 'ux', visual: 'ux', visuals: 'ux', feedback: 'ux', navigation: 'ux', polish: 'ux', design: 'ux',
+  crash: 'bug', error: 'bug', defect: 'bug', broken: 'bug', functional: 'bug', functionality: 'bug', network: 'bug',
+  perf: 'performance', lag: 'performance', fps: 'performance', loading: 'performance',
+  a11y: 'accessibility',
+  level: 'level_design', map: 'level_design', 'level design': 'level_design', 'level-design': 'level_design',
+  text: 'content', story: 'content', narrative: 'content', writing: 'content', copy: 'content', audio: 'content', art: 'content',
+  economy: 'balance', difficulty: 'balance', progression: 'balance'
+});
+
+/** The severity an out of range one becomes. */
+export const NORMALISED_SEVERITY = 'minor';
+
+/**
+ * The category an unknown one becomes: `other` when the schema has it, else a
+ * catch-all the schema names, else `ux`. The server's schema today has no
+ * catch-all (bug, ux, balance, content, performance, accessibility,
+ * level_design), so an unknown word that no synonym places becomes `ux`, the
+ * category most unplaced findings turn out to be.
+ *
+ * @param {Array<string>} categories the schema's enum
+ * @returns {string}
+ */
+export function catchAllCategory(categories) {
+  for (const candidate of ['other', 'misc', 'general', 'uncategorized']) if (categories.includes(candidate)) return candidate;
+  return categories.includes('ux') ? 'ux' : categories[0];
+}
+
+/**
+ * The enum lists and defaults of the report schema's root and of one finding,
+ * read from the schema itself so a pack update moves them.
+ *
+ * @param {Object} schema the playtest-report schema
+ * @returns {{severities: Array<string>, categories: Array<string>, root: Object, finding: Object}}
+ */
+export function enumFields(schema) {
+  const props = (schema && schema.properties) || {};
+  const item = (props.findings && props.findings.items && props.findings.items.properties) || {};
+  const enumsOf = properties => Object.fromEntries(Object.entries(properties)
+    .filter(([, spec]) => spec && Array.isArray(spec.enum))
+    .map(([key, spec]) => [key, { values: spec.enum, ...(spec.default !== undefined ? { default: spec.default } : {}) }]));
+  return {
+    severities: (item.severity && item.severity.enum) || [],
+    categories: (item.category && item.category.enum) || [],
+    root: enumsOf(props),
+    finding: enumsOf(item)
+  };
+}
+
+/**
+ * Bring known out of range values back into the schema before upload,
+ * instead of losing the run to them. The server's schema stays strict; this
+ * only rewrites what the model's own words say plainly:
+ *
+ * - a value that differs from an allowed one only in case or spacing takes
+ *   the allowed spelling (`Major` is `major`);
+ * - a severity that is a category word (`"severity": "ux"`) moves to
+ *   `category` when the category is missing or not allowed, and the severity
+ *   becomes `minor`; a severity that is anything else unknown becomes `minor`;
+ * - a category that is a severity word, with the severity out of range,
+ *   swaps back; any other unknown category becomes a synonym the schema has,
+ *   else `other` (or the catch-all, see {@link catchAllCategory});
+ * - any other enum field, at the root or on a finding, with a value the
+ *   schema does not allow takes the schema's `default` when it has one.
+ *
+ * Every change is returned, and recorded on the finding as a runner note at
+ * the end of its description. The Texas round lost a run to `"severity": "ux"`
+ * when the model's budget ran out before its repair turn.
+ *
+ * @param {Object} report
+ * @param {Object} schema the playtest-report schema
+ * @returns {{report: Object, changes: Array<{id: string|null, field: string, from: any, to: any, note: string}>}}
+ */
+export function normaliseEnums(report, schema) {
+  if (!report || typeof report !== 'object' || !schema) return { report, changes: [] };
+  const fields = enumFields(schema);
+  const changes = [];
+  const spelled = (value, allowed) => {
+    if (typeof value !== 'string') return null;
+    const key = value.trim().toLowerCase().replace(/[\s-]+/g, '_');
+    return allowed.find(entry => String(entry).toLowerCase() === key) ?? null;
+  };
+  const fixEnum = (target, key, spec, id) => {
+    if (!(key in target) || spec.values.includes(target[key])) return;
+    const respelled = spelled(target[key], spec.values);
+    const to = respelled !== null ? respelled : spec.default;
+    if (to === undefined) return;
+    changes.push({ id, field: key, from: target[key], to, note: `${key} ${JSON.stringify(target[key])} is not allowed, so it became ${JSON.stringify(to)}` });
+    target[key] = to;
+  };
+
+  const root = { ...report };
+  for (const [key, spec] of Object.entries(fields.root)) fixEnum(root, key, spec, null);
+
+  const { severities, categories } = fields;
+  if (Array.isArray(root.findings)) {
+    root.findings = root.findings.map(original => {
+      if (!original || typeof original !== 'object' || Array.isArray(original)) return original;
+      const finding = { ...original };
+      const id = typeof finding.id === 'string' ? finding.id : null;
+      const notes = [];
+      const record = (field, from, to, note) => {
+        changes.push({ id, field, from, to, note });
+        notes.push(note);
+      };
+      if (severities.length > 0 && categories.length > 0) {
+        const severityOk = severities.includes(finding.severity);
+        const categoryOk = categories.includes(finding.category);
+        const severityRespelled = severityOk ? finding.severity : spelled(finding.severity, severities);
+        const categoryRespelled = categoryOk ? finding.category : spelled(finding.category, categories);
+        const severityAsCategory = severityRespelled === null ? spelled(finding.severity, categories) : null;
+        const categoryAsSeverity = categoryRespelled === null ? spelled(finding.category, severities) : null;
+        if (severityRespelled !== null && !severityOk) record('severity', finding.severity, severityRespelled, `severity ${JSON.stringify(finding.severity)} became ${JSON.stringify(severityRespelled)}`);
+        if (categoryRespelled !== null && !categoryOk) record('category', finding.category, categoryRespelled, `category ${JSON.stringify(finding.category)} became ${JSON.stringify(categoryRespelled)}`);
+        let severity = severityRespelled;
+        let category = categoryRespelled;
+        if (severity === null && category === null && severityAsCategory !== null && categoryAsSeverity !== null) {
+          // The two were written in each other's slot.
+          record('severity', finding.severity, categoryAsSeverity, `severity ${JSON.stringify(finding.severity)} and category ${JSON.stringify(finding.category)} were swapped, so they were put back`);
+          record('category', finding.category, severityAsCategory, `category ${JSON.stringify(finding.category)} is a severity word, so it moved to severity`);
+          severity = categoryAsSeverity;
+          category = severityAsCategory;
+        }
+        if (severity === null && severityAsCategory !== null && category === null) {
+          record('category', finding.category === undefined ? null : finding.category, severityAsCategory, `severity ${JSON.stringify(finding.severity)} is a category, so it moved to category`);
+          category = severityAsCategory;
+        }
+        if (severity === null) {
+          record('severity', finding.severity === undefined ? null : finding.severity, NORMALISED_SEVERITY, `severity ${JSON.stringify(finding.severity === undefined ? null : finding.severity)} is not a severity, so it became ${NORMALISED_SEVERITY}`);
+          severity = NORMALISED_SEVERITY;
+        }
+        if (category === null) {
+          const word = typeof finding.category === 'string' ? finding.category.trim().toLowerCase() : '';
+          const synonym = CATEGORY_SYNONYMS[word] && categories.includes(CATEGORY_SYNONYMS[word]) ? CATEGORY_SYNONYMS[word] : null;
+          const to = synonym || catchAllCategory(categories);
+          record('category', finding.category === undefined ? null : finding.category, to, `category ${JSON.stringify(finding.category === undefined ? null : finding.category)} is not a category, so it became ${to}`);
+          category = to;
+        }
+        finding.severity = severity;
+        finding.category = category;
+      }
+      for (const [key, spec] of Object.entries(fields.finding)) {
+        if (key === 'severity' || key === 'category') continue;
+        const before = changes.length;
+        fixEnum(finding, key, spec, id);
+        if (changes.length > before) notes.push(changes[changes.length - 1].note);
+      }
+      if (notes.length > 0) {
+        const line = `Runner note: ${notes.join('; ')}.`;
+        const description = typeof finding.description === 'string' ? finding.description : '';
+        finding.description = description ? `${description}\n\n${line}` : line;
+      }
+      return finding;
+    });
+  }
+  return { report: changes.length > 0 ? root : report, changes };
+}
+
+/** How report.md lists what the runner normalised. */
+export const NORMALISED_HEADING = '## Normalised by the runner (values the report schema does not allow)';
+
+/**
+ * report.md agreeing with report.json about normalised values: each change is
+ * listed under {@link NORMALISED_HEADING}, replacing a list written before.
+ *
+ * @param {string} markdown
+ * @param {Array<{id: string|null, note: string}>} changes
+ * @returns {string}
+ */
+export function appendNormalisedMarkdown(markdown, changes) {
+  if (!Array.isArray(changes) || changes.length === 0) return markdown;
+  let body = String(markdown || '');
+  const at = body.indexOf(NORMALISED_HEADING);
+  if (at !== -1) {
+    // A list written before is replaced: from its heading to the next
+    // second level heading, or the end.
+    const next = body.indexOf('\n## ', at + NORMALISED_HEADING.length);
+    body = body.slice(0, at) + (next === -1 ? '' : body.slice(next + 1));
+  }
+  body = body.replace(/\s+$/, '');
+  const lines = changes.map(change => `- ${change.id ? `${change.id}: ` : ''}${change.note}.`);
+  return `${body}\n\n${NORMALISED_HEADING}\n${lines.join('\n')}\n`;
+}
+
 /**
  * Check every `screenshot` evidence ref against the files actually on disk.
  *
@@ -301,6 +486,8 @@ export function repairInstruction(details, evidenceProblems, missingSections = [
 
 export default {
   createReportValidator,
+  normaliseEnums,
+  appendNormalisedMarkdown,
   stampReport,
   checkEvidence,
   normaliseEvidence,

package/src/run/resume.js

@@ -149,6 +149,8 @@ export async function resumeJob({ api, gameId, jobId, job, runs, state, repoRoot
     build: { kind: driverName === 'playwright_web' ? 'url' : 'godot_project', ref: target },
     repo: { commit: '', dirty: false },
     max_actions: (journal.options && journal.options.maxActions) ?? null,
+    // The same body as the first register, so a long job replays as long.
+    ...(journal.options?.runLength === 'long' ? { run_length: 'long' } : {}),
     pack_versions: { pack: pack.pack_version }
   }, { idempotencyKey: journal.idempotency_key });
 
@@ -165,6 +167,9 @@ export async function resumeJob({ api, gameId, jobId, job, runs, state, repoRoot
     ...options,
     controller: journal.options?.controller || options.controller || 'standard',
     maxActions: (journal.options && journal.options.maxActions) ?? options.maxActions,
+    // The journal's answer, never the flag: a job is long or standard for good.
+    runLength: journal.options?.runLength === 'long' ? 'long' : 'standard',
+    ...(Number.isInteger(journal.options?.longRunFactor) ? { longRunFactor: journal.options.longRunFactor } : {}),
     concurrency: options.concurrency ?? (journal.options && journal.options.concurrency),
     uploadVideo: options.uploadVideo ?? (journal.options && journal.options.uploadVideo),
     uploadTranscript: options.uploadTranscript ?? (journal.options && journal.options.uploadTranscript)

package/src/run/tools.js

@@ -83,7 +83,7 @@ export const PLAY_TOOLS = Object.freeze([
       additionalProperties: false,
       properties: {
         ref: { type: 'string', description: 'playwright_web only, the preferred target: the ref shown on the element, such as e12.' },
-        target: { type: 'string', description: 'web: button "Start" or Start Game. Godot: a node path such as /root/Main/UI/StartButton.' },
+        target: { type: 'string', description: 'web: button "Start" or Start Game; a whole line copied from the screen also works, and is clicked by its ref, else its @(x,y). Text that is not on the screen is refused at once. Godot: a node path such as /root/Main/UI/StartButton.' },
         x: { type: 'number' },
         y: { type: 'number' },
         thought_in_character: thought,

package/src/runDefaults.js

@@ -0,0 +1,90 @@
+/**
+ * What a plain `run` asks for, in one place, so `brief`, `check` and `run`
+ * quote the same job. Kept free of runner imports: `check` and `brief` load
+ * this without pulling in the drivers.
+ */
+
+/**
+ * Personas used when the developer names none. One since 0.3.2: a first run
+ * should cost one persona run, and a second persona is one `--personas` away.
+ * Spec 17 defaulted to two, which made the default run cost twice what most
+ * customers expected to pay for a first look.
+ */
+export const DEFAULT_PERSONAS = Object.freeze(['curious-kid']);
+
+/** The modules `run` registers. `aggregate_report` is free and buys the aggregate's steps. */
+export const RUN_MODULES = Object.freeze(['persona_playtest', 'aggregate_report']);
+
+/**
+ * Actions a sign up form alone took in customer runs (an email, a password, a
+ * confirm, a name, the submit, and the clicks between them), which is why a
+ * brief's test account matters on a short action budget.
+ */
+export const SIGN_UP_ACTIONS = 8;
+
+/**
+ * The default persona set as far as this pack has them: the defaults it
+ * carries, else its first persona.
+ *
+ * @param {Array<{slug: string}>} [available]
+ * @returns {Array<string>}
+ */
+export function defaultPersonasFor(available = null) {
+  if (!Array.isArray(available)) return [...DEFAULT_PERSONAS];
+  const slugs = available.map(persona => persona && persona.slug).filter(Boolean);
+  const defaults = DEFAULT_PERSONAS.filter(slug => slugs.includes(slug));
+  return defaults.length > 0 ? defaults : slugs.slice(0, 1);
+}
+
+/** "1 persona", "3 personas". */
+export function personaCount(count) {
+  return `${count} persona${count === 1 ? '' : 's'}`;
+}
+
+/**
+ * An estimate's breakdown in words, every unit named with its own price:
+ * "profile job $4.00, a persona run $1.00 each (x2)".
+ *
+ * @param {Array<{unit: string, count: number, unit_cents: number}>} breakdown
+ * @param {(cents: number) => string} money
+ * @returns {string}
+ */
+export function describeBreakdown(breakdown, money) {
+  const parts = [];
+  for (const line of Array.isArray(breakdown) ? breakdown : []) {
+    if (!line || !(Number(line.count) > 0)) continue;
+    const price = money(line.unit_cents);
+    const times = Number(line.count) > 1 ? ` (x${line.count})` : '';
+    if (line.unit === 'playtest_run') parts.push(`a persona run ${price} each${times}`);
+    else if (line.unit === 'playtest_run_long') parts.push(`a long persona run ${price} each${times}`);
+    else if (line.unit === 'playtest_profile') parts.push(`profile job ${price}${times}`);
+    else if (line.unit === 'playtest_fix') parts.push(`a fix attempt ${price} each${times}`);
+    else parts.push(`${line.unit} ${price}${times}`);
+  }
+  return parts.join(', ');
+}
+
+/**
+ * The persona slugs in a list that the pack does not have.
+ *
+ * @param {Array<string>} slugs
+ * @param {Array<{slug: string}>} available
+ * @returns {Array<string>}
+ */
+export function unknownPersonas(slugs, available) {
+  const known = new Set((available || []).map(persona => persona && persona.slug));
+  return (slugs || []).filter(slug => !known.has(slug));
+}
+
+/**
+ * "unknown persona "x"; the valid slugs are a, b and c. "ravensight-playtest personas" lists them."
+ *
+ * @param {Array<string>} unknown
+ * @param {Array<{slug: string}>} available
+ * @returns {string}
+ */
+export function unknownPersonaMessage(unknown, available) {
+  const slugs = (available || []).map(persona => persona.slug);
+  const listed = slugs.length > 1 ? `${slugs.slice(0, -1).join(', ')} and ${slugs[slugs.length - 1]}` : slugs.join('');
+  return `unknown persona${unknown.length === 1 ? '' : 's'} ${unknown.map(slug => JSON.stringify(slug)).join(', ')}; the valid slugs are ${listed || '(none in this pack)'}. "ravensight-playtest personas" lists them.`;
+}

package/src/simulation/index.js

@@ -0,0 +1,176 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+import { boundedString, bridgeOptions, observation, openBridge } from './protocol.js';
+const hash = value => createHash('sha256').update(JSON.stringify(value)).digest('hex');
+const MAX_TRACE_BYTES = 4 * 1024 * 1024;
+const MAX_ACTIONS = 1000000;
+const compareText = (a, b) => a < b ? -1 : a > b ? 1 : 0;
+export function validateConfig(input) {
+  if (!input || input.version !== 1 || !boundedString(input.seed, 128)) throw new Error('Simulation config needs version 1 and a seed of 1 to 128 characters.');
+  for (const [key, max] of [['runs', 10000], ['max_steps', 10000], ['workers', 16]]) if (!Number.isInteger(input[key]) || input[key] < 1 || input[key] > max) throw new Error(`Invalid simulation ${key}.`);
+  if (!Array.isArray(input.strategies) || !input.strategies.length || input.strategies.length > 16) throw new Error('Choose 1 to 16 simulation strategies.');
+  const strategies = input.strategies.map(strategy => {
+    if (!strategy || typeof strategy.name !== 'string' || !/^[a-z][a-z0-9_-]{0,63}$/.test(strategy.name) || !['random', 'scripted', 'scored'].includes(strategy.policy)) throw new Error('Invalid simulation strategy.');
+    if (strategy.policy === 'scripted' && (!Array.isArray(strategy.actions) || !strategy.actions.length || strategy.actions.length > 10000 || strategy.actions.some(id => !boundedString(id, 128)))) throw new Error('Scripted strategy needs 1 to 10000 action IDs.');
+    return { name: strategy.name, policy: strategy.policy, ...(strategy.policy === 'scripted' ? { actions: [...strategy.actions] } : {}) };
+  });
+  if (new Set(strategies.map(s => s.name)).size !== strategies.length) throw new Error('Strategy names must be unique.');
+  const config = { version: 1, seed: input.seed, runs: input.runs, max_steps: input.max_steps, workers: input.workers, strategies };
+  if (Buffer.byteLength(JSON.stringify(config)) > 2 * 1024 * 1024) throw new Error('Simulation configuration exceeds 2 MiB.');
+  return config;
+}
+export function configHash(config) { return hash({ version: 1, seed: config.seed, runs: config.runs, max_steps: config.max_steps, workers: config.workers, strategies: config.strategies.map(({ name, policy }) => ({ name, policy })) }); }
+export function deriveSeed(seed, index) { return hash([seed, index]); }
+function choose(state, strategy, index, seed) {
+  if (strategy.policy === 'scripted') {
+    const id = strategy.actions[index];
+    if (!id) throw new Error('Scripted strategy exhausted before terminal state.');
+    if (!state.actions.some(action => action.id === id)) throw new Error('Scripted strategy selected an illegal action.');
+    return id;
+  }
+  if (strategy.policy === 'scored') {
+    const choices = state.actions.filter(action => Number.isFinite(action.scores?.[strategy.name]));
+    if (!choices.length) throw new Error('Game supplied no scores for this strategy.');
+    return choices.sort((a, b) => b.scores[strategy.name] - a.scores[strategy.name] || compareText(a.id, b.id))[0].id;
+  }
+  const choices = [...state.actions].sort((a, b) => compareText(a.id, b.id));
+  return choices[parseInt(hash([seed, index]).slice(0, 12), 16) % choices.length].id;
+}
+function stateRecord(state) { return { done: state.done, state_hash: state.state_hash, ...(state.outcome === undefined ? {} : { outcome: state.outcome }), ...(state.metrics === undefined ? {} : { metrics: state.metrics }) }; }
+async function writeJson(file, value) { await fs.writeFile(file, JSON.stringify(value) + '\n', { flag: 'wx', mode: 0o600 }); }
+export async function probeBridge({ bridge, seed = 'preflight', strategy = 'random', max_steps = 1, signal }) {
+  if (!boundedString(seed, 128) || !boundedString(strategy, 80) || !Number.isInteger(max_steps) || max_steps < 1 || max_steps > 10000) throw new Error('Invalid simulation probe.');
+  const client = openBridge(bridge, { signal });
+  try { return await client.request('simulation.start', { version: 1, seed, strategy, max_steps }); }
+  finally { await client.stop(); }
+}
+export async function runBatch({ config: rawConfig, bridge: rawBridge, outputDir, playtest, signal, onProgress }) {
+  const config = validateConfig(rawConfig); const bridge = bridgeOptions(rawBridge);
+  if (typeof outputDir !== 'string' || !outputDir) throw new Error('Simulation output directory is required.');
+  outputDir = path.resolve(outputDir); await fs.mkdir(outputDir, { recursive: true });
+  const resultsPath = path.join(outputDir, 'results.jsonl'); const file = await fs.open(resultsPath, 'wx', 0o600);
+  const started = Date.now(); const controller = new AbortController(); let stopReason = null;
+  const cancel = () => { stopReason ||= 'cancelled'; controller.abort(); };
+  signal?.addEventListener('abort', cancel, { once: true }); if (signal?.aborted) cancel();
+  const timer = setTimeout(() => { stopReason ||= 'batch_deadline'; controller.abort(); }, bridge.batchTimeoutMs);
+  let next = 0, reservedSteps = 0, writeChain = Promise.resolve(), commitTail = Promise.resolve();
+  const traces = [];
+  const summary = { version: 1, seed: config.seed, config_hash: configHash(config), requested_runs: config.runs, completed_runs: 0, failed_runs: 0, truncated_runs: 0, total_steps: 0, strategies: config.strategies.map(s => ({ name: s.name, runs: 0, completed: 0, failed: 0, truncated: 0, outcomes: {}, metrics: {} })), trace_count: 0, duration_ms: 0 };
+  async function execute(index, priorCommit) {
+    const strategy = config.strategies[index % config.strategies.length]; const seed = deriveSeed(config.seed, index);
+    const row = { version: 1, run_index: index, seed, strategy: strategy.name, status: 'failed', steps: 0 };
+    const trace = index < 10 ? { version: 1, run_index: index, seed, strategy: strategy.name, max_steps: config.max_steps, local_config_hash: hash(config), states: [], actions: [] } : null;
+    let traceBytes = 1024, client, pendingAction, phase = 'start';
+    const record = state => {
+      if (!trace) return;
+      const item = stateRecord(state); traceBytes += Buffer.byteLength(JSON.stringify(item)) + 1;
+      if (traceBytes > MAX_TRACE_BYTES) { trace.truncated = true; return; }
+      trace.states.push(item);
+    };
+    try {
+      if (controller.signal.aborted) throw new Error('Simulation stopped before run started.');
+      client = openBridge(bridge, { signal: controller.signal, playtest });
+      let state = await client.request('simulation.start', { version: 1, seed, strategy: strategy.name, max_steps: config.max_steps }); record(state);
+      while (!state.done && row.steps < config.max_steps) {
+        if (reservedSteps >= MAX_ACTIONS) { stopReason ||= 'action_budget'; controller.abort(); throw new Error('Batch action budget exhausted.'); }
+        phase = 'policy';
+        const action = choose(state, strategy, row.steps, seed); reservedSteps++;
+        pendingAction = action; phase = 'step';
+        state = await client.request('simulation.step', { action }); pendingAction = undefined; row.steps++; summary.total_steps++;
+        if (trace && !trace.truncated) { trace.actions.push(action); traceBytes += Buffer.byteLength(JSON.stringify(action)) + 1; }
+        record(state);
+      }
+      row.status = state.done ? 'completed' : 'truncated';
+      if (state.done) { row.outcome = state.outcome || 'unspecified'; row.metrics = state.metrics || {}; }
+    } catch (error) { row.error = error.message.slice(0, 200); if (trace) { trace.failure_phase = phase; if (pendingAction) trace.failed_action = pendingAction; } }
+    finally { if (client) await client.stop(); }
+    // Commit in run-index order, retaining at most one result per worker. This
+    // makes floating-point aggregate order independent of process scheduling.
+    await priorCommit;
+    const aggregate = summary.strategies[index % config.strategies.length]; aggregate.runs++;
+    summary[`${row.status}_runs`]++; aggregate[row.status]++;
+    if (row.status === 'completed') {
+      // Bound aggregate cardinality even if a malicious adapter invents a new
+      // metric/outcome name in every run. Raw per-run values remain on disk.
+      const outcome = Object.hasOwn(aggregate.outcomes, row.outcome) || Object.keys(aggregate.outcomes).length < 127 ? row.outcome : 'other';
+      aggregate.outcomes[outcome] = (Object.hasOwn(aggregate.outcomes, outcome) ? aggregate.outcomes[outcome] : 0) + 1;
+      for (const [key, value] of Object.entries(row.metrics)) {
+        if (!Object.hasOwn(aggregate.metrics, key) && Object.keys(aggregate.metrics).length >= 32) continue;
+        if (!Object.hasOwn(aggregate.metrics, key)) aggregate.metrics[key] = { count: 0, min: value, max: value, mean: 0 };
+        const metric = aggregate.metrics[key];
+        metric.count++; metric.min = Math.min(metric.min, value); metric.max = Math.max(metric.max, value);
+        // Weighted arithmetic avoids overflow from subtracting opposite extremes.
+        metric.mean = Math.max(metric.min, Math.min(metric.max, metric.mean * ((metric.count - 1) / metric.count) + value / metric.count));
+      }
+    }
+    if (trace) { if (trace.truncated) trace.actions = trace.actions.slice(0, Math.max(0, trace.states.length - 1)); trace.status = row.status; trace.error = row.error; const target = path.join(outputDir, `trace-${String(index).padStart(5, '0')}.json`); await writeJson(target, trace); traces.push(target); }
+    writeChain = writeChain.then(() => file.write(JSON.stringify(row) + '\n')); await writeChain;
+    if (onProgress) await onProgress({ finished_runs: summary.completed_runs + summary.failed_runs + summary.truncated_runs, requested_runs: config.runs, completed_runs: summary.completed_runs, failed_runs: summary.failed_runs, truncated_runs: summary.truncated_runs, total_steps: summary.total_steps });
+  }
+  try {
+    const workers = Array.from({ length: Math.min(config.workers, config.runs) }, async () => {
+      try {
+        while (next < config.runs) {
+          const index = next++; const prior = commitTail; let release;
+          commitTail = new Promise(resolve => { release = resolve; });
+          try { await execute(index, prior); } finally { release(); }
+        }
+      }
+      catch (error) { controller.abort(); throw error; }
+    });
+    const outcomes = await Promise.allSettled(workers);
+    const failure = outcomes.find(item => item.status === 'rejected');
+    if (failure) throw failure.reason;
+  }
+  finally { clearTimeout(timer); signal?.removeEventListener('abort', cancel); controller.abort(); try { await writeChain; } finally { await file.close(); } }
+  summary.trace_count = traces.length; summary.duration_ms = Math.min(bridge.batchTimeoutMs, Date.now() - started);
+  traces.sort();
+  const report = { results_path: resultsPath, stop_reason: stopReason, determinism_note: 'Replay verifies recorded observations. Reproducibility requires the game to honor its seed; this runner cannot prove determinism.' };
+  await writeJson(path.join(outputDir, 'summary.json'), summary);
+  return { summary, traces, report };
+}
+export async function replayTrace({ trace, bridge, signal, outputDir }) {
+  if (typeof trace === 'string') { if ((await fs.stat(trace)).size > MAX_TRACE_BYTES + 1024) throw new Error('Trace exceeds replay limit.'); trace = JSON.parse(await fs.readFile(trace, 'utf8')); }
+  const failedStart = trace?.status === 'failed' && trace?.failure_phase === 'start' && trace?.states?.length === 0;
+  if (!trace || trace.version !== 1 || !boundedString(trace.seed, 128) || !boundedString(trace.strategy, 80) || !Number.isInteger(trace.max_steps) || trace.max_steps < 1 || trace.max_steps > 10000 || !Array.isArray(trace.actions) || trace.actions.length > 10000 || trace.actions.some(action => !boundedString(action, 128)) || !Array.isArray(trace.states) || (!failedStart && trace.states.length !== trace.actions.length + 1) || (failedStart && trace.actions.length) || (trace.failed_action !== undefined && !boundedString(trace.failed_action, 128))) throw new Error('Trace is invalid for replay.');
+  if (Buffer.byteLength(JSON.stringify(trace)) > MAX_TRACE_BYTES + 1024) throw new Error('Trace exceeds replay limit.');
+  for (const state of trace.states) observation({ ...state, actions: state.done ? [] : [{ id: 'replay-validation' }] });
+  const options = bridgeOptions(bridge);
+  const deadline = AbortSignal.timeout(options.batchTimeoutMs);
+  const replaySignal = signal ? AbortSignal.any([signal, deadline]) : deadline;
+  const client = openBridge(options, { signal: replaySignal });
+  let result = { matched: true, matched_prefix: false, fully_replayed: false, failure_reproduced: null, first_divergence: null };
+  const start = () => client.request('simulation.start', { version: 1, seed: trace.seed, strategy: trace.strategy, max_steps: trace.max_steps });
+  async function reproduce(request, step) {
+    try {
+      const actual = await request(); result.matched = false; result.failure_reproduced = false;
+      result.first_divergence = { step, reason: 'Recorded protocol failure did not recur.', actual: stateRecord(actual) };
+    } catch (error) {
+      result.failure_reproduced = error.message === trace.error;
+      result.matched = result.failure_reproduced;
+      if (!result.matched) result.first_divergence = { step, expected_error: trace.error, actual_error: error.message };
+    }
+  }
+  try {
+    if (failedStart) await reproduce(start, 0);
+    else {
+      for (let i = 0; i < trace.states.length; i++) {
+        let state;
+        try { state = i === 0 ? await start() : await client.request('simulation.step', { action: trace.actions[i - 1] }); }
+        catch (error) { result.matched = false; result.first_divergence = { step: i, actual_error: error.message }; break; }
+        const expected = trace.states[i]; const actual = stateRecord(state);
+        const canonical = value => JSON.stringify([value.state_hash, value.done, value.outcome ?? null, Object.entries(value.metrics || {}).sort(([a], [b]) => compareText(a, b))]);
+        if (canonical(expected) !== canonical(actual)) { result.matched = false; result.first_divergence = { step: i, expected, actual }; break; }
+        if (i < trace.actions.length && !state.actions.some(action => action.id === trace.actions[i])) { result.matched = false; result.first_divergence = { step: i, reason: 'Recorded action is no longer legal.' }; break; }
+      }
+      if (result.matched) {
+        result.matched_prefix = true;
+        result.fully_replayed = !trace.truncated && trace.status !== 'failed';
+        if (trace.status === 'failed' && trace.failed_action && !trace.truncated) await reproduce(() => client.request('simulation.step', { action: trace.failed_action }), trace.states.length);
+      }
+    }
+  } finally { await client.stop(); }
+  if (outputDir) { await fs.mkdir(outputDir, { recursive: true }); const file = path.resolve(outputDir, 'replay.json'); await writeJson(file, result); result.report_path = file; }
+  return result;
+}

package/src/simulation/protocol.js

@@ -0,0 +1,90 @@
+import { spawn } from 'node:child_process';
+import path from 'node:path';
+const forbidden = new Set(['__proto__', 'constructor', 'prototype']);
+export const MAX_RESPONSE = 256 * 1024;
+export function boundedString(value, max) { return typeof value === 'string' && value.length > 0 && value.length <= max && !forbidden.has(value); }
+export function observation(value) {
+  if (!value || typeof value !== 'object' || Array.isArray(value) || typeof value.done !== 'boolean' || !boundedString(value.state_hash, 128) || !Array.isArray(value.actions) || value.actions.length > 256) throw new Error('Invalid simulation observation.');
+  const numbers = (map, limit, maxAbsolute = Infinity) => {
+    if (!map || typeof map !== 'object' || Array.isArray(map) || Object.keys(map).length > limit || Object.entries(map).some(([key, n]) => !boundedString(key, 80) || typeof n !== 'number' || (!Number.isFinite(n) || Math.abs(n) > maxAbsolute))) throw new Error('Invalid simulation numeric map.');
+    return { ...map };
+  };
+  const actions = value.actions.map(action => {
+    if (!action || !boundedString(action.id, 128)) throw new Error('Invalid simulation action.');
+    return { id: action.id, ...(action.scores === undefined ? {} : { scores: numbers(action.scores, 16) }) };
+  });
+  if (new Set(actions.map(action => action.id)).size !== actions.length) throw new Error('Duplicate simulation action.');
+  if (!value.done && !actions.length) throw new Error('Nonterminal simulation needs legal actions.');
+  if (value.outcome !== undefined && !boundedString(value.outcome, 80)) throw new Error('Invalid simulation outcome.');
+  return { done: value.done, state_hash: value.state_hash, actions, ...(value.metrics === undefined ? {} : { metrics: numbers(value.metrics, 32, 1e15) }), ...(value.outcome === undefined ? {} : { outcome: value.outcome }) };
+}
+export function bridgeOptions(bridge) {
+  // Native Windows has no POSIX process-group kill. Do not buy a batch whose
+  // adapter descendants we cannot reliably stop; WSL uses the Linux path.
+  if (process.platform === 'win32') throw new Error('Simulation Runner is not supported on native Windows. Run it in WSL, Linux or macOS so cancellation can stop the adapter process group.');
+  if (!bridge || !path.isAbsolute(bridge.command || '') || (bridge.args !== undefined && (!Array.isArray(bridge.args) || bridge.args.length > 256 || bridge.args.some(arg => typeof arg !== 'string' || arg.length > 8192))) || (bridge.cwd !== undefined && !path.isAbsolute(bridge.cwd))) throw new Error('Bridge needs an absolute command, string arguments and optional absolute cwd.');
+  const timeoutMs = bridge.timeoutMs ?? 30000; const batchTimeoutMs = bridge.batchTimeoutMs ?? 900000;
+  if (!Number.isInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 30000 || !Number.isInteger(batchTimeoutMs) || batchTimeoutMs < 1 || batchTimeoutMs > 900000) throw new Error('Invalid simulation deadline.');
+  if (bridge.env !== undefined && (!bridge.env || typeof bridge.env !== 'object' || Array.isArray(bridge.env) || Object.entries(bridge.env).some(([k, v]) => !boundedString(k, 128) || typeof v !== 'string' || v.length > 8192))) throw new Error('Invalid bridge environment.');
+  return { ...bridge, timeoutMs, batchTimeoutMs };
+}
+export function childEnvironment(extra = {}, playtest = {}) {
+  // Local adapters are trusted code, not a security sandbox. Avoid accidentally
+  // forwarding account/provider credentials or inherited synthetic identities.
+  const env = {};
+  for (const [key, value] of Object.entries({ ...process.env, ...extra })) {
+    if (/TOKEN|SECRET|PASSWORD|CREDENTIAL|API.?KEY|PRIVATE.?KEY|AUTH|^AWS_|^AZURE_|^GOOGLE_|^ANTHROPIC_|^OPENAI_|^NODE_OPTIONS$|^RAVENSIGHT_.*(?:RUN|JOB)/i.test(key)) continue;
+    env[key] = value;
+  }
+  env.RAVENSIGHT_PLAYTEST_TOKEN = typeof playtest.token === 'string' && playtest.token ? playtest.token : 'invalid-simulation-preflight-token';
+  env.RAVENSIGHT_PLAYTEST_RUN_ID = String(playtest.runId || 'local-simulation');
+  env.RAVENSIGHT_PLAYTEST_JOB_ID = String(playtest.jobId || 'local-simulation');
+  return env;
+}
+export function openBridge(raw, { signal, playtest } = {}) {
+  const bridge = bridgeOptions(raw);
+  const child = spawn(bridge.command, bridge.args || [], { cwd: bridge.cwd, shell: false, detached: process.platform !== 'win32', env: childEnvironment(bridge.env, playtest), stdio: ['pipe', 'pipe', 'pipe'] });
+  let buffer = Buffer.alloc(0), id = 0, pending, failure, closing;
+  const kill = sig => { try { if (process.platform !== 'win32' && child.pid) process.kill(-child.pid, sig); else child.kill(sig); } catch {} };
+  const fail = message => { failure ||= new Error(message); if (pending) { clearTimeout(pending.timer); pending.reject(failure); pending = null; } };
+  const stop = () => {
+    if (closing) return closing;
+    fail('Simulation bridge stopped.'); signal?.removeEventListener('abort', abort);
+    kill('SIGTERM');
+    // Always escalate against the process group, including when child.killed is
+    // true or the leader exited while a descendant still owns its pipes.
+    closing = new Promise(resolve => setTimeout(() => { kill('SIGKILL'); child.stdin.destroy(); child.stdout.destroy(); child.stderr.destroy(); resolve(); }, 100));
+    return closing;
+  };
+  const abort = () => { fail('Simulation cancelled.'); void stop(); };
+  signal?.addEventListener('abort', abort, { once: true });
+  if (signal?.aborted) abort();
+  child.stderr.resume();
+  child.on('error', () => { fail('Simulation bridge could not start.'); void stop(); });
+  child.on('exit', () => { fail('Simulation bridge exited.'); });
+  child.stdin.on('error', () => { fail('Simulation bridge input failed.'); void stop(); });
+  child.stdout.on('data', chunk => {
+    buffer = Buffer.concat([buffer, chunk]);
+    if (buffer.length > MAX_RESPONSE) { fail('Simulation response exceeds 256 KB.'); void stop(); return; }
+    let end;
+    while ((end = buffer.indexOf(10)) >= 0) {
+      const line = buffer.subarray(0, end); buffer = buffer.subarray(end + 1);
+      try {
+        const message = JSON.parse(line.toString('utf8'), (key, value) => { if (forbidden.has(key)) throw new Error(); return value; });
+        if (!pending || message.jsonrpc !== '2.0' || message.id !== pending.id || message.error || !Object.hasOwn(message, 'result')) throw new Error();
+        const result = observation(message.result); const entry = pending; pending = null; clearTimeout(entry.timer); entry.resolve(result);
+      } catch { fail('Invalid simulation protocol response.'); void stop(); return; }
+    }
+  });
+  return {
+    request(method, params) {
+      if (failure) return Promise.reject(failure);
+      if (pending) return Promise.reject(new Error('Simulation request already pending.'));
+      return new Promise((resolve, reject) => {
+        const requestId = ++id;
+        pending = { id: requestId, resolve, reject, timer: setTimeout(() => { fail('Simulation response timed out.'); void stop(); }, bridge.timeoutMs) };
+        child.stdin.write(JSON.stringify({ jsonrpc: '2.0', id: requestId, method, params }) + '\n');
+      });
+    }, stop
+  };
+}

package/src/state/index.js

@@ -100,6 +100,9 @@ export async function listStates(options = {}) {
       state
     });
   }
-  found.sort((a, b) => String(b.updated_at).localeCompare(String(a.updated_at)));
+  // Two journals written in the same millisecond tie on updated_at; the job
+  // id breaks the tie (descending, like the time), so the order never
+  // depends on how the filesystem lists the directory.
+  found.sort((a, b) => String(b.updated_at).localeCompare(String(a.updated_at)) || String(b.job_id).localeCompare(String(a.job_id)));
   return found;
 }

package/src/ui/index.js

@@ -184,6 +184,19 @@ export async function choose(question, labels, options = {}) {
   }
 }
 
+/**
+ * Bold on a terminal that shows it, plain everywhere else (a pipe, CI, a
+ * log file, NO_COLOR), so a saved log never carries escape codes.
+ *
+ * @param {string} text
+ * @param {{isTTY?: boolean}} [target] the stream it will be written to
+ * @returns {string}
+ */
+export function bold(text, target = stream) {
+  if (process.env.NO_COLOR || !(target && target.isTTY)) return String(text);
+  return `\u001b[1m${text}\u001b[22m`;
+}
+
 /** Cents as dollars, the one place that formatting exists. */
 export function money(cents) {
   if (cents === null || cents === undefined) return 'unknown';
@@ -219,5 +232,6 @@ export const ui = {
   confirm,
   choose,
   money,
-  bytes
+  bytes,
+  bold
 };