ravensight-playtest 0.1.0

package/src/run/tools.js

@@ -0,0 +1,344 @@
+import { OBSERVATION_MAX_CHARS, truncateObservation } from './drivers/driver.js';
+
+/**
+ * The client tool set for `persona_playtest.act`.
+ *
+ * **The names are not ours to choose.** `routing.json`'s
+ * `persona_playtest.act` lists `allowed_client_tools: ["snapshot", "click",
+ * "type", "key", "navigate", "wait", "screenshot"]`, and the proxy answers
+ * 400 `tool_not_allowed` for anything else. Two consequences worth knowing
+ * before editing this file:
+ *
+ * - There is no `note` tool and no `observe`/`act` pair, so the log note
+ *   rides along as an optional field on whichever tool the persona is
+ *   already calling. Same information, same transcript line, no tool the
+ *   server would refuse.
+ * - There is no `quit` tool either. A persona that wants to stop simply
+ *   stops calling tools; the loop reads a turn with no tool call as
+ *   `persona_quit` and keeps the model's own words as the quit detail.
+ *
+ * Spec 03's accounting is kept: one action is one input to the game, so
+ * `click`, `type`, `key` and `navigate` count against `max_actions` while
+ * `snapshot`, `screenshot` and `wait` do not.
+ */
+
+/** Note kinds, from spec 03's `log_note`. */
+export const NOTE_KINDS = Object.freeze(['bug', 'confusing', 'good', 'suggestion', 'reaction']);
+
+/** Severities, from the report schema's finding severity enum. */
+export const SEVERITIES = Object.freeze(['blocker', 'major', 'minor', 'cosmetic', 'praise']);
+
+/** Tools that spend an action. */
+export const ACTION_TOOLS = Object.freeze(['click', 'type', 'key', 'navigate']);
+
+const thought = {
+  type: 'string',
+  description: 'One to three sentences in the persona voice, first person. Never break character here.'
+};
+
+const note = {
+  type: 'object',
+  description: 'Optional: something worth recording at this step. Engineer voice, not persona voice. Use it for anything surprising, frustrating, delightful, broken or confusing.',
+  additionalProperties: false,
+  required: ['kind', 'text'],
+  properties: {
+    kind: { type: 'string', enum: [...NOTE_KINDS] },
+    text: { type: 'string' },
+    severity: { type: 'string', enum: [...SEVERITIES] }
+  }
+};
+
+/** The tool definitions sent with every `persona_playtest.act` call. */
+export const PLAY_TOOLS = Object.freeze([
+  {
+    name: 'snapshot',
+    description: 'Look at the current screen. Does not spend an action. Use it after anything that might have changed the game state.',
+    input_schema: {
+      type: 'object',
+      additionalProperties: false,
+      properties: { thought_in_character: thought, note }
+    }
+  },
+  {
+    name: 'click',
+    description: 'Click one thing. Spends an action. Either name the target or give x and y pixel coordinates. What a named target means depends on the driver: on playwright_web it is an accessibility node such as button "Start", a visible text fragment, or a CSS selector; on godot_driver it is a node path from the scene tree you were shown.',
+    input_schema: {
+      type: 'object',
+      additionalProperties: false,
+      properties: {
+        target: { type: 'string', description: 'web: button "Start", Start Game, or #start-button. Godot: a node path such as /root/Main/UI/StartButton.' },
+        x: { type: 'number' },
+        y: { type: 'number' },
+        thought_in_character: thought,
+        note
+      }
+    }
+  },
+  {
+    name: 'type',
+    description: 'Type text. Spends an action. Set submit to press Enter afterwards.',
+    input_schema: {
+      type: 'object',
+      additionalProperties: false,
+      required: ['text'],
+      properties: {
+        text: { type: 'string' },
+        submit: { type: 'boolean' },
+        thought_in_character: thought,
+        note
+      }
+    }
+  },
+  {
+    name: 'key',
+    description: 'Press one key. Spends an action. Names are the usual ones: Enter, Space, ArrowUp, Escape, a, 1.',
+    input_schema: {
+      type: 'object',
+      additionalProperties: false,
+      required: ['key'],
+      properties: {
+        key: { type: 'string' },
+        thought_in_character: thought,
+        note
+      }
+    }
+  },
+  {
+    name: 'navigate',
+    description: 'Go to a URL inside this build. Spends an action. Anything outside the build origin is refused, so do not guess URLs.',
+    input_schema: {
+      type: 'object',
+      additionalProperties: false,
+      required: ['url'],
+      properties: {
+        url: { type: 'string' },
+        thought_in_character: thought,
+        note
+      }
+    }
+  },
+  {
+    name: 'wait',
+    description: 'Wait for the game to catch up. Does not spend an action. Capped, so ask for what you need rather than a long sleep.',
+    input_schema: {
+      type: 'object',
+      additionalProperties: false,
+      required: ['ms'],
+      properties: {
+        ms: { type: 'integer', minimum: 0 },
+        thought_in_character: thought,
+        note
+      }
+    }
+  },
+  {
+    name: 'screenshot',
+    description: 'Save a screenshot, for evidence. Does not spend an action. Only at key moments: a screen or mode seen for the first time, a bug, a strong reaction, a crash, the final state. Never on a fixed interval and never the same state twice.',
+    input_schema: {
+      type: 'object',
+      additionalProperties: false,
+      required: ['name', 'reason'],
+      properties: {
+        name: { type: 'string', description: 'a short slug, such as title-screen or level-2-softlock' },
+        reason: { type: 'string' },
+        thought_in_character: thought,
+        note
+      }
+    }
+  }
+]);
+
+/** The single tool for `persona_playtest.report`. */
+export const REPORT_TOOLS = Object.freeze([
+  {
+    name: 'write_file',
+    description: 'Write one of the two report files. path must be exactly report.md or report.json.',
+    input_schema: {
+      type: 'object',
+      additionalProperties: false,
+      required: ['path', 'content'],
+      properties: {
+        path: { type: 'string', enum: ['report.md', 'report.json'] },
+        content: { type: 'string' }
+      }
+    }
+  }
+]);
+
+/**
+ * @param {Object} toolUse a tool_use block from ./model.js
+ * @returns {{kind: string}|null} the driver action, or null for the tools
+ *   that do not touch the game
+ */
+export function actionFor(toolUse) {
+  const input = toolUse.input || {};
+  switch (toolUse.name) {
+    case 'click':
+      return Number.isFinite(Number(input.x)) && Number.isFinite(Number(input.y))
+        ? { kind: 'click', target: { x: Number(input.x), y: Number(input.y) } }
+        : { kind: 'click', target: String(input.target || '') };
+    case 'type':
+      return { kind: 'type', text: String(input.text || ''), submit: Boolean(input.submit) };
+    case 'key':
+      return { kind: 'key', key: String(input.key || '') };
+    case 'navigate':
+      return { kind: 'navigate', url: String(input.url || '') };
+    case 'wait':
+      return { kind: 'wait', ms: Number(input.ms) || 0 };
+    default:
+      return null;
+  }
+}
+
+/**
+ * Execute one tool call against the driver.
+ *
+ * The result is a `tool_result` content block plus the bookkeeping the loop
+ * needs: whether an action was spent, whether the screen moved, and the
+ * observation to keep in context.
+ *
+ * @param {Object} args
+ * @param {Object} args.driver
+ * @param {Object} args.toolUse
+ * @param {(note: Object) => void} [args.onNote]
+ * @param {(shot: Object) => void} [args.onScreenshot]
+ * @returns {Promise<{result: Object, observation: Object|null, spentAction: boolean, changed: boolean, error: string|null}>}
+ */
+export async function executeTool({ driver, toolUse, onNote, onScreenshot }) {
+  const input = toolUse.input || {};
+  if (toolUse.malformed) {
+    return {
+      result: toolResult(toolUse.id, 'That tool call did not parse as JSON. Send it again, smaller.', true),
+      observation: null,
+      spentAction: false,
+      changed: false,
+      error: 'malformed_tool_input'
+    };
+  }
+
+  if (input.note && onNote) onNote({ ...input.note, tool: toolUse.name });
+
+  if (toolUse.name === 'screenshot') {
+    try {
+      const shot = await driver.screenshot(String(input.name || 'shot'));
+      if (onScreenshot) onScreenshot({ ...shot, reason: String(input.reason || '') });
+      return {
+        result: toolResult(toolUse.id, `Saved ${shot.relativePath}.`),
+        observation: null,
+        spentAction: false,
+        changed: false,
+        error: null
+      };
+    } catch (error) {
+      return {
+        result: toolResult(toolUse.id, `The screenshot failed: ${message(error)}`, true),
+        observation: null,
+        spentAction: false,
+        changed: false,
+        error: message(error)
+      };
+    }
+  }
+
+  if (toolUse.name === 'snapshot') {
+    const observation = await driver.observe();
+    return {
+      result: observationResult(toolUse.id, observation),
+      observation,
+      spentAction: false,
+      changed: false,
+      error: null
+    };
+  }
+
+  const action = actionFor(toolUse);
+  if (!action) {
+    return {
+      result: toolResult(toolUse.id, `There is no tool called ${toolUse.name}.`, true),
+      observation: null,
+      spentAction: false,
+      changed: false,
+      error: 'unknown_tool'
+    };
+  }
+
+  const outcome = await driver.act(action);
+  const observation = await driver.observe();
+  const spentAction = ACTION_TOOLS.includes(toolUse.name) && outcome.ok;
+  const prefix = outcome.ok
+    ? (outcome.changed ? '' : 'That did not change the screen.\n')
+    : `That did not work: ${outcome.error}\n`;
+  return {
+    result: observationResult(toolUse.id, observation, prefix, !outcome.ok),
+    observation,
+    spentAction,
+    changed: Boolean(outcome.changed),
+    error: outcome.ok ? null : String(outcome.error || 'act failed')
+  };
+}
+
+/**
+ * A tool_result whose content is the new observation, with the screenshot
+ * attached as an image when the driver decided the text was too thin to
+ * play from. Images are pruned from history by the caller: the act step
+ * accepts a very small number of image blocks per request.
+ *
+ * @param {string} id
+ * @param {Object} observation
+ * @param {string} [prefix]
+ * @param {boolean} [isError]
+ */
+export function observationResult(id, observation, prefix = '', isError = false) {
+  const content = [
+    {
+      type: 'text',
+      text: `${prefix}step ${observation.step}${observation.url ? ` (${observation.url})` : ''}${citationLine(observation)}\n${truncateObservation(observation.text, OBSERVATION_MAX_CHARS)}`
+    }
+  ];
+  if (observation.screenshotBase64) {
+    content.push({
+      type: 'image',
+      source: {
+        type: 'base64',
+        media_type: observation.screenshotMediaType || 'image/png',
+        data: observation.screenshotBase64
+      }
+    });
+  }
+  return { type: 'tool_result', tool_use_id: id, content, ...(isError ? { is_error: true } : {}) };
+}
+
+/**
+ * @param {string} id
+ * @param {string} text
+ * @param {boolean} [isError]
+ */
+export function toolResult(id, text, isError = false) {
+  return { type: 'tool_result', tool_use_id: id, content: [{ type: 'text', text }], ...(isError ? { is_error: true } : {}) };
+}
+
+/**
+ * What the persona can cite as `file_line` evidence.
+ *
+ * The Godot driver reports the current scene and the scenes and scripts
+ * behind it, which is the only way a finding from a running game can name a
+ * `res://` path. The dogfood gate asks for exactly that ("at least one
+ * finding cites a res:// scene or script path"), so the paths have to reach
+ * the model, not just the transcript.
+ *
+ * @param {Object} observation
+ * @returns {string}
+ */
+export function citationLine(observation) {
+  const parts = [];
+  if (observation.scene) parts.push(`scene ${observation.scene}`);
+  if (Array.isArray(observation.scenes) && observation.scenes.length > 0) parts.push(`scenes ${observation.scenes.slice(0, 8).join(', ')}`);
+  if (Array.isArray(observation.scripts) && observation.scripts.length > 0) parts.push(`scripts ${observation.scripts.slice(0, 8).join(', ')}`);
+  return parts.length > 0 ? `\n[${parts.join(' | ')}] (cite these as file_line evidence)` : '';
+}
+
+function message(error) {
+  return String(error && error.message ? error.message : error);
+}
+
+export default { PLAY_TOOLS, REPORT_TOOLS, executeTool, actionFor, toolResult, observationResult };

package/src/run/transcript.js

@@ -0,0 +1,93 @@
+import { createWriteStream } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
+import path from 'node:path';
+import { RUN_FILES } from './paths.js';
+import { scanForSecrets } from './secretScan.js';
+
+/**
+ * `transcript.jsonl`, one JSON object per line.
+ *
+ * Two things depend on the exact contents:
+ *
+ * 1. `regenerate` rebuilds `report.md` and `report.json` from the transcript
+ *    plus the screenshots, with the game not running (spec 03). So every
+ *    input the report step reads has to be in here: the header line, each
+ *    observation, each action, each note, each screenshot name.
+ * 2. Evidence refs are written as `step N`, where N is the step field on
+ *    these lines. A transcript that renumbers on resume would invalidate
+ *    every evidence ref in the report, so a resumed run starts a new
+ *    transcript in a new run directory rather than appending.
+ *
+ * The file is opt-in for upload (spec 17 section 1.7) and the server expires
+ * it at 90 days, so it is written in full locally either way.
+ *
+ * Every line is scanned before it is written, because the transcript is the
+ * one artifact whose content the CLI does not author: it carries whatever the
+ * game printed and whatever the model quoted back. A leaked key would
+ * otherwise sit in a file the developer may then opt into uploading, and the
+ * server's own scan only ever sees the report. A hit is refused and replaced
+ * by a line naming the shapes, never the value, and the run continues: losing
+ * one transcript line is better than losing the playthrough, and the
+ * `secret_withheld` line is what tells anyone reading it that something was
+ * dropped.
+ */
+export function createTranscript(runDirectory) {
+  const target = path.join(runDirectory, RUN_FILES.transcript);
+  /** @type {import('node:fs').WriteStream|null} */
+  let stream = null;
+  let lines = 0;
+  let withheld = 0;
+
+  return {
+    path: target,
+
+    async open() {
+      await mkdir(runDirectory, { recursive: true });
+      stream = createWriteStream(target, { flags: 'a' });
+      return this;
+    },
+
+    /**
+     * @param {Object} line
+     */
+    write(line) {
+      if (!stream) throw new Error('transcript is not open');
+      const body = JSON.stringify({ t: Date.now(), ...line });
+      const secrets = scanForSecrets(body);
+      lines += 1;
+      if (secrets.length > 0) {
+        withheld += 1;
+        // The shapes, never the value: this file is the last place a found
+        // secret should be written out.
+        stream.write(`${JSON.stringify({
+          t: Date.now(),
+          kind: 'secret_withheld',
+          of: line.kind || 'unknown',
+          shapes: secrets.map(secret => `${secret.type} (${secret.count})`)
+        })}\n`);
+        return;
+      }
+      stream.write(`${body}\n`);
+    },
+
+    get lineCount() {
+      return lines;
+    },
+
+    /** How many lines were refused for looking secret shaped. */
+    get withheldCount() {
+      return withheld;
+    },
+
+    async close() {
+      if (!stream) return;
+      const closing = stream;
+      stream = null;
+      await new Promise((resolve, reject) => {
+        closing.end(error => (error ? reject(error) : resolve()));
+      });
+    }
+  };
+}
+
+export default createTranscript;

package/src/run/usage.js

@@ -0,0 +1,115 @@
+import { writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { RUN_FILES } from './paths.js';
+import { assertNoSecrets } from './secretScan.js';
+
+/**
+ * Token accounting for one run, and `usage.json`.
+ *
+ * Tokens are counted locally, off the stream: `message_start` and
+ * `message_delta` carry them, and the proxy re-emits both verbatim.
+ *
+ * **Dollars are not counted locally, and cannot be.** Per-token prices live
+ * in `routing.json`, which is server-only by design (CLAUDE.md: no model id
+ * or price anywhere in the API's `src/**`, and the client subset
+ * `clientRouting()` deliberately carries neither). So `total_usd` is taken
+ * from the server, which prices every call as it makes it: the run's
+ * `usage.total_usd` comes back on every heartbeat answer
+ * (`PATCH /jobs/:jobId/runs/:runId` -> `{ run: { usage: { total_usd, calls,
+ * wall_seconds } } }`). A locally invented figure would be a second,
+ * disagreeing number for the one thing the customer is paying, which is
+ * worse than no number.
+ *
+ * `by_model[].usd` therefore stays 0: the server reports the run total, not
+ * a per-model split, and splitting it here would be arithmetic we made up.
+ * The model ids themselves are real, read off `message_start.message.model`.
+ */
+export function createUsageLedger() {
+  /** @type {Map<string, {model: string, input_tokens: number, cache_read_tokens: number, output_tokens: number, usd: number}>} */
+  const byModel = new Map();
+  const started = Date.now();
+  let calls = 0;
+  let serverTotalUsd = null;
+  let serverCalls = null;
+
+  return {
+    /**
+     * @param {{model: string|null, usage: Object}} turn
+     */
+    record({ model, usage }) {
+      calls += 1;
+      const key = model || 'unknown';
+      const entry = byModel.get(key) || { model: key, input_tokens: 0, cache_read_tokens: 0, output_tokens: 0, usd: 0 };
+      // The report schema has no cache-creation field, so creation tokens
+      // fold into input_tokens (they are billed as input) and only cache
+      // reads are broken out. The PoC did the same.
+      entry.input_tokens += Number(usage.input_tokens || 0) + Number(usage.cache_creation_input_tokens || 0);
+      entry.cache_read_tokens += Number(usage.cache_read_input_tokens || 0);
+      entry.output_tokens += Number(usage.output_tokens || 0);
+      byModel.set(key, entry);
+    },
+
+    /**
+     * The authoritative spend, from a heartbeat or run read.
+     *
+     * @param {{total_usd?: number, calls?: number}|null|undefined} runUsage
+     */
+    recordServerUsage(runUsage) {
+      if (!runUsage) return;
+      if (Number.isFinite(Number(runUsage.total_usd))) serverTotalUsd = Number(runUsage.total_usd);
+      if (Number.isFinite(Number(runUsage.calls))) serverCalls = Number(runUsage.calls);
+    },
+
+    get modelCalls() {
+      return calls;
+    },
+
+    /**
+     * The `usage` object as the report schema wants it.
+     *
+     * @returns {{by_model: Array, total_usd: number, wall_seconds: number}}
+     */
+    toReportUsage() {
+      return {
+        by_model: [...byModel.values()],
+        total_usd: serverTotalUsd === null ? 0 : Number(serverTotalUsd.toFixed(6)),
+        wall_seconds: Number(((Date.now() - started) / 1000).toFixed(3))
+      };
+    },
+
+    /**
+     * `usage.json`, which is not schema-constrained and so can say where
+     * each number came from.
+     *
+     * @returns {Object}
+     */
+    toUsageFile() {
+      const report = this.toReportUsage();
+      return {
+        ...report,
+        local_model_calls: calls,
+        server_model_calls: serverCalls,
+        usd_source: serverTotalUsd === null ? 'unavailable' : 'server',
+        note: 'Token counts are read off the model stream. total_usd is the server figure, because per-token prices are server side only.'
+      };
+    }
+  };
+}
+
+/**
+ * @param {string} runDirectory
+ * @param {ReturnType<typeof createUsageLedger>} ledger
+ * @returns {Promise<string>} the path written
+ */
+export async function writeUsageFile(runDirectory, ledger) {
+  const target = path.join(runDirectory, RUN_FILES.usageJson);
+  const body = `${JSON.stringify(ledger.toUsageFile(), null, 2)}\n`;
+  // Token counts and model ids should never look secret shaped, and that is
+  // the point of checking: this file is uploaded by default, so anything
+  // unexpected in it is refused rather than shipped.
+  assertNoSecrets('usage.json', body);
+  await writeFile(target, body, 'utf8');
+  return target;
+}
+
+export default { createUsageLedger, writeUsageFile };

package/src/state/index.js

@@ -0,0 +1,105 @@
+import { readdir } from 'node:fs/promises';
+import { paths } from '../paths.js';
+import { readJson, writeJson } from '../fsutil.js';
+import { CliError } from '../errors.js';
+
+export const STATE_VERSION = 1;
+
+/**
+ * The resumable job journal, one file per job, at
+ * `<repo>/.ravensight/jobs/<jobId>/state.json`.
+ *
+ * Written atomically (temp file plus rename, see fsutil), because the whole
+ * point of the file is to survive the process being killed, and a half written
+ * journal is worse than none: `resume` would read it, believe a run finished
+ * that did not, and skip it.
+ *
+ * The contents beyond `job_id` and `game_id` belong to the runner. cli-core
+ * reads those two, plus `runs` when it is present, and treats everything else
+ * as opaque so a runner change never needs a change here.
+ */
+
+/** The skeleton a runner starts from. */
+export function newState({ jobId, gameId, ...rest }) {
+  return {
+    state_version: STATE_VERSION,
+    job_id: jobId,
+    game_id: gameId,
+    created_at: new Date().toISOString(),
+    updated_at: new Date().toISOString(),
+    runs: {},
+    ...rest
+  };
+}
+
+export const stateFileFor = (jobId, options = {}) => paths.stateFile(jobId, options);
+
+/**
+ * Read a job's journal, or null when there is none.
+ * @param {string} jobId
+ * @param {{repoRoot?: string}} [options]
+ * @returns {Promise<Object|null>}
+ */
+export async function readState(jobId, options = {}) {
+  return readJson(paths.stateFile(jobId, options), null);
+}
+
+/**
+ * Write a job's journal. `updated_at` is stamped here rather than by every
+ * caller, so a journal always says when it was last touched even if the writer
+ * forgot.
+ * @param {string} jobId
+ * @param {Object} state
+ * @param {{repoRoot?: string}} [options]
+ */
+export async function writeState(jobId, state, options = {}) {
+  const next = { ...state, state_version: STATE_VERSION, updated_at: new Date().toISOString() };
+  await writeJson(paths.stateFile(jobId, options), next);
+  return next;
+}
+
+/**
+ * Read, transform, write. Not a lock: one job is driven by one process, and two
+ * processes on one job is a mistake no amount of locking here would make safe.
+ * @param {string} jobId
+ * @param {(state: Object|null) => Object} mutate
+ * @param {{repoRoot?: string}} [options]
+ */
+export async function updateState(jobId, mutate, options = {}) {
+  const current = await readState(jobId, options);
+  const next = mutate(current);
+  if (!next || typeof next !== 'object') {
+    throw new CliError('updateState needs an object back from its mutator.');
+  }
+  return writeState(jobId, next, options);
+}
+
+/**
+ * Every job this repo has a journal for, newest first.
+ * @param {{repoRoot?: string}} [options]
+ * @returns {Promise<Array<{job_id: string, game_id: string, updated_at: string, state: Object}>>}
+ */
+export async function listStates(options = {}) {
+  let entries;
+  try {
+    entries = await readdir(paths.jobsDir(options), { withFileTypes: true });
+  } catch (error) {
+    if (error.code === 'ENOENT') return [];
+    throw error;
+  }
+
+  const found = [];
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    const state = await readState(entry.name, options);
+    if (!state) continue;
+    found.push({
+      job_id: state.job_id || entry.name,
+      game_id: state.game_id || null,
+      updated_at: state.updated_at || null,
+      state
+    });
+  }
+  found.sort((a, b) => String(b.updated_at).localeCompare(String(a.updated_at)));
+  return found;
+}