ravensight-playtest 0.1.0

package/src/run/model.js

@@ -0,0 +1,384 @@
+import Anthropic from '@anthropic-ai/sdk';
+
+/**
+ * Every model call this CLI makes goes through here, and every one of them
+ * goes to Ravensight's proxy rather than to the Anthropic API.
+ *
+ * What that means concretely, and why:
+ *
+ * - **No Anthropic key exists on the developer's machine.** The client is
+ *   built with `authToken` (the `gt_cli_...` token from the device-code
+ *   flow) and an explicitly null `apiKey`, so a stray `ANTHROPIC_API_KEY`
+ *   in the developer's environment is never picked up and never sent
+ *   anywhere.
+ * - **No `model` field, ever.** Routing is the server's decision
+ *   (`src/playtest/packs/routing.json`, server-only). The proxy rebuilds
+ *   the body from an allowlist and supplies the model itself; a body that
+ *   names a real routed model id is refused 403 `model_not_allowed`. The
+ *   CLI cannot know which model a step resolves to, which is the point.
+ * - **Client tools only.** The proxy refuses `mcp_servers`, `container`,
+ *   `betas` and `service_tier` outright, refuses any tool whose type starts
+ *   `web_search`, `computer`, `bash` or `text_editor`, and refuses any tool
+ *   whose name is not in the step's own `allowed_client_tools`. So the tool
+ *   names in ./tools.js are not a style choice, they are the contract.
+ * - **Streaming is forced on by the proxy.** Asking for a non-stream
+ *   response would get a stream anyway, so we ask for one.
+ * - **`X-Playtest-Job`, `-Run` and `-Step` are required on every call.**
+ *   The step has to belong to the run's own module (plus the shared
+ *   `brief.validate`), or the answer is 400 `step_not_allowed`.
+ *
+ * The two answers that change what the caller does next:
+ *
+ * - `X-Playtest-Budget: warn` comes back at or past 80 percent of the run's
+ *   model budget. It is surfaced to the developer and it tells the persona
+ *   loop to wrap up and write its report while there is still money to do
+ *   it with.
+ * - `402 run_budget_exceeded` is the end of the run's spending. Partial
+ *   results are written and uploaded, and the process exits 2.
+ */
+
+/** Where the proxy lives under the API host. */
+export const MODEL_BASE_PATH = '/api/v1/playtest/model';
+
+/** Step names, from routing.json by way of `clientRouting()`. */
+export const STEPS = Object.freeze({
+  personaAct: 'persona_playtest.act',
+  personaReport: 'persona_playtest.report',
+  profileTriage: 'game_profile.triage',
+  profileSynthesis: 'game_profile.synthesis',
+  aggregateDedup: 'aggregate.dedup',
+  aggregateSynthesis: 'aggregate.synthesis',
+  briefValidate: 'brief.validate'
+});
+
+/** Default retry budget for the answers that are worth retrying. */
+export const MAX_ATTEMPTS = 4;
+
+/** Codes the proxy raises that a retry can actually clear. */
+const RETRYABLE_CODES = new Set(['concurrency_limited', 'velocity_limited', 'rate_limited', 'upstream_error']);
+
+export class ProxyError extends Error {
+  /**
+   * @param {{status: number, code: string, message: string, retryAfter?: number|null, figures?: Object}} init
+   */
+  constructor({ status, code, message, retryAfter = null, figures = {} }) {
+    super(message);
+    this.name = 'ProxyError';
+    this.status = status;
+    this.code = code;
+    this.retryAfter = retryAfter;
+    this.figures = figures;
+  }
+
+  /** The run has no model budget left. Write what we have and exit 2. */
+  get isBudget() {
+    return this.status === 402;
+  }
+
+  /** The job or run is closed, cancelled or past its wall clock. Exit 10. */
+  get isClosed() {
+    return this.status === 409;
+  }
+
+  /** Bad or missing credentials. Exit 4. */
+  get isAuth() {
+    return this.status === 401 || this.status === 403;
+  }
+
+  get isRetryable() {
+    if (this.status === 502) return true;
+    return this.status === 429 && RETRYABLE_CODES.has(this.code);
+  }
+}
+
+/** Every retry was spent and the proxy still will not answer. Exit 5. */
+export class ModelUnavailableError extends Error {
+  constructor(message, cause) {
+    super(message);
+    this.name = 'ModelUnavailableError';
+    this.cause = cause;
+  }
+}
+
+/**
+ * Read a header off whatever the SDK handed back: a `Headers` object on a
+ * response, a plain object on some error paths, or nothing at all.
+ *
+ * @param {any} headers
+ * @param {string} name
+ * @returns {string|null}
+ */
+export function headerValue(headers, name) {
+  if (!headers) return null;
+  if (typeof headers.get === 'function') return headers.get(name);
+  const lower = name.toLowerCase();
+  for (const [key, value] of Object.entries(headers)) {
+    if (String(key).toLowerCase() === lower) return String(value);
+  }
+  return null;
+}
+
+/**
+ * Normalise the two error envelopes the playtest surface uses.
+ *
+ * The router's own refusals are the Anthropic envelope the SDK already
+ * parses, `{ error: { type, message, ...figures } }`. The two middleware
+ * refusals (no `gt_cli_` credential, an ingest key) keep the repo's flat
+ * `{ error: 'code', message }` shape, because they come from the shared CLI
+ * auth chain. Both have to be understood here or a 401 reads as an
+ * unknown failure.
+ *
+ * @param {any} error an error thrown by the SDK
+ * @returns {ProxyError}
+ */
+export function toProxyError(error) {
+  const status = Number(error && error.status) || 0;
+  const body = (error && error.error) || {};
+  const inner = body && typeof body.error === 'object' && body.error !== null ? body.error : null;
+  const flat = typeof body.error === 'string' ? body.error : null;
+  const code = (inner && inner.type) || flat || (error && error.name) || 'model_error';
+  const message = (inner && inner.message) || body.message || (error && error.message) || 'The model proxy refused the call.';
+  const headerRetry = Number(headerValue(error && error.headers, 'retry-after'));
+  const bodyRetry = Number(inner && inner.retry_after);
+  const retryAfter = Number.isFinite(bodyRetry) && bodyRetry > 0
+    ? bodyRetry
+    : (Number.isFinite(headerRetry) && headerRetry > 0 ? headerRetry : null);
+  const figures = {};
+  if (inner) {
+    for (const [key, value] of Object.entries(inner)) {
+      if (key !== 'type' && key !== 'message') figures[key] = value;
+    }
+  }
+  return new ProxyError({ status, code, message, retryAfter, figures });
+}
+
+/**
+ * @param {Object} opts
+ * @param {string} opts.apiUrl e.g. https://api.ravensight.io
+ * @param {string} opts.token the gt_cli_ token
+ * @param {number} [opts.timeoutMs]
+ * @param {Function} [opts.fetch] test seam
+ * @returns {Anthropic}
+ */
+export function createModelClient({ apiUrl, token, timeoutMs = 300000, fetch: fetchImpl } = {}) {
+  if (!apiUrl) throw new Error('createModelClient needs an apiUrl');
+  if (!token) throw new Error('createModelClient needs a gt_cli_ token');
+  return new Anthropic({
+    baseURL: `${String(apiUrl).replace(/\/+$/, '')}${MODEL_BASE_PATH}`,
+    authToken: token,
+    // Explicitly null so the SDK never falls back to a local
+    // ANTHROPIC_API_KEY. This CLI has no business holding one.
+    apiKey: null,
+    // Retries are ours: the proxy's 429s carry a code and a retry_after
+    // that decide whether waiting will help at all, and the SDK cannot see
+    // that difference.
+    maxRetries: 0,
+    timeout: timeoutMs,
+    ...(fetchImpl ? { fetch: fetchImpl } : {})
+  });
+}
+
+/**
+ * Merge one usage payload into an accumulator. `message_delta.usage` is
+ * cumulative rather than incremental, so the last value of each field wins
+ * instead of being added (the proxy does the same thing on its side, and
+ * the two figures have to agree).
+ *
+ * @param {Object} accumulator
+ * @param {Object|null|undefined} usage
+ */
+export function mergeUsage(accumulator, usage) {
+  if (!usage || typeof usage !== 'object') return accumulator;
+  for (const field of ['input_tokens', 'cache_creation_input_tokens', 'cache_read_input_tokens', 'output_tokens']) {
+    const value = Number(usage[field]);
+    if (Number.isFinite(value)) accumulator[field] = value;
+  }
+  return accumulator;
+}
+
+/**
+ * One model turn.
+ *
+ * @param {Object} args
+ * @param {Anthropic} args.client
+ * @param {string} args.jobId
+ * @param {string} args.runId
+ * @param {string} args.step a routing.json step name
+ * @param {Array} args.messages
+ * @param {string|Array} [args.system]
+ * @param {Array} [args.tools]
+ * @param {number} args.maxTokens asked for, clamped by the step's own ceiling
+ * @param {boolean} [args.escalate] honoured only on an escalatable step
+ * @param {AbortSignal} [args.signal]
+ * @returns {Promise<{text: string, toolUses: Array, stopReason: string|null, usage: Object, model: string|null, tier: string|null, budget: string|null}>}
+ */
+export async function callStep({
+  client,
+  jobId,
+  runId,
+  step,
+  messages,
+  system,
+  tools,
+  maxTokens,
+  escalate = false,
+  signal
+}) {
+  const headers = {
+    'X-Playtest-Job': jobId,
+    'X-Playtest-Run': runId,
+    'X-Playtest-Step': step
+  };
+  if (escalate) headers['X-Playtest-Escalate'] = '1';
+
+  const body = { messages, max_tokens: maxTokens, stream: true };
+  if (system !== undefined) body.system = system;
+  if (Array.isArray(tools) && tools.length > 0) body.tools = tools;
+
+  let stream;
+  let response;
+  try {
+    const answered = await client.messages.create(body, { headers, signal }).withResponse();
+    stream = answered.data;
+    response = answered.response;
+  } catch (error) {
+    if (error && (error.name === 'AbortError' || error.name === 'APIUserAbortError')) throw error;
+    throw toProxyError(error);
+  }
+
+  const tier = headerValue(response && response.headers, 'x-playtest-model-tier');
+  const budget = headerValue(response && response.headers, 'x-playtest-budget');
+
+  const usage = { input_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0, output_tokens: 0 };
+  const blocks = new Map();
+  let text = '';
+  let stopReason = null;
+  let model = null;
+
+  try {
+    await consume(stream);
+  } catch (error) {
+    if (error && (error.name === 'AbortError' || error.name === 'APIUserAbortError')) throw error;
+    // The SDK's own iterator raises an `event: error` frame as an APIError
+    // with no status: the status line was sent long ago. Everything from the
+    // first byte onwards is therefore a 502 by construction, which is also
+    // how the proxy classifies its own mid-stream failures.
+    const proxied = toProxyError(error);
+    throw proxied.status ? proxied : new ProxyError({
+      status: 502,
+      code: proxied.code === 'model_error' ? 'upstream_error' : proxied.code,
+      message: proxied.message
+    });
+  }
+
+  async function consume(events) {
+    for await (const event of events) {
+    switch (event.type) {
+      case 'message_start':
+        // The proxy re-emits the upstream events verbatim, so this is where
+        // the resolved model id becomes visible even though the tier header
+        // deliberately does not carry it. It is recorded in usage.json for
+        // reconciliation against the server's own figures, never used to
+        // pick a model.
+        model = (event.message && event.message.model) || model;
+        mergeUsage(usage, event.message && event.message.usage);
+        break;
+      case 'content_block_start':
+        blocks.set(event.index, { block: event.content_block, json: '' });
+        break;
+      case 'content_block_delta': {
+        const entry = blocks.get(event.index);
+        if (!entry) break;
+        if (event.delta && event.delta.type === 'text_delta') {
+          text += event.delta.text;
+        } else if (event.delta && event.delta.type === 'input_json_delta') {
+          entry.json += event.delta.partial_json || '';
+        }
+        break;
+      }
+      case 'message_delta':
+        if (event.delta && event.delta.stop_reason) stopReason = event.delta.stop_reason;
+        mergeUsage(usage, event.usage);
+        break;
+      default:
+        // Anything else, `ping` included, is not a frame this loop has an
+        // opinion about. The SDK drops `ping` before it reaches here and
+        // raises an `error` frame as a thrown APIError, which the caller
+        // above converts.
+        break;
+    }
+    }
+  }
+
+  const toolUses = [];
+  for (const { block, json } of blocks.values()) {
+    if (!block || block.type !== 'tool_use') continue;
+    let input = block.input && Object.keys(block.input).length > 0 ? block.input : {};
+    if (json) {
+      try {
+        input = JSON.parse(json);
+      } catch {
+        // A truncated tool call (usually `max_tokens` cutting the JSON in
+        // half) is reported as a malformed call so the caller can tell the
+        // model rather than crash on it.
+        toolUses.push({ id: block.id, name: block.name, input: null, malformed: true, raw: json });
+        continue;
+      }
+    }
+    toolUses.push({ id: block.id, name: block.name, input, malformed: false });
+  }
+
+  return { text, toolUses, stopReason, usage, model, tier, budget };
+}
+
+/**
+ * @param {number} ms
+ * @param {Function} [sleep]
+ */
+function wait(ms, sleep) {
+  if (sleep) return sleep(ms);
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * `callStep` with the retries that are worth taking, and none of the ones
+ * that are not.
+ *
+ * Retried: 502 `upstream_error`, and a 429 whose code says the wait is the
+ * fix (`concurrency_limited`, `velocity_limited`, `rate_limited`). The
+ * proxy's own `retry_after` is honoured when it sends one, since it knows
+ * how long its hourly window or its upstream asked for.
+ *
+ * Never retried: 402 (no money left, waiting changes nothing), 409 (the job
+ * is closed), 401 and 403 (credentials or scope), 400 and 404 (a contract
+ * problem in this CLI, and hammering it just spends the developer's
+ * patience). 429 `count_tokens_limited` is not retried either: the job's
+ * free-count ceiling does not refill inside one job.
+ *
+ * @param {Object} args see callStep, plus:
+ * @param {number} [args.maxAttempts]
+ * @param {(info: Object) => void} [args.onRetry]
+ * @param {Function} [args.sleep] test seam
+ */
+export async function callStepWithRetry(args) {
+  const { maxAttempts = MAX_ATTEMPTS, onRetry, sleep, ...rest } = args;
+  let last = null;
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    try {
+      return await callStep(rest);
+    } catch (error) {
+      if (error instanceof ProxyError && !error.isRetryable) throw error;
+      if (error && (error.name === 'AbortError' || error.name === 'APIUserAbortError')) throw error;
+      last = error;
+      if (attempt === maxAttempts) break;
+      const suggested = error instanceof ProxyError && error.retryAfter ? error.retryAfter * 1000 : null;
+      const backoff = suggested ?? Math.min(1000 * 2 ** (attempt - 1), 30000);
+      if (onRetry) onRetry({ attempt, maxAttempts, waitMs: backoff, error });
+      await wait(backoff, sleep);
+    }
+  }
+  const reason = last instanceof ProxyError ? `${last.status} ${last.code}: ${last.message}` : String(last && last.message);
+  throw new ModelUnavailableError(`The model proxy did not answer after ${maxAttempts} attempts (${reason}).`, last);
+}
+
+export default { createModelClient, callStep, callStepWithRetry, STEPS, ProxyError, ModelUnavailableError };

package/src/run/paths.js

@@ -0,0 +1,88 @@
+import path from 'node:path';
+
+/**
+ * The local layout, spec 17 section 1.5:
+ *
+ *   <repo>/.ravensight/
+ *     config.json                     (cli-core: game id, driver defaults)
+ *     jobs/<job_id>/state.json        the resumable journal (./state.js)
+ *     jobs/<job_id>/capability-report.json
+ *     jobs/<job_id>/aggregate-report.md
+ *     jobs/<job_id>/aggregate-report.json
+ *     jobs/<job_id>/runs/<run_id>/report.md
+ *     jobs/<job_id>/runs/<run_id>/report.json
+ *     jobs/<job_id>/runs/<run_id>/usage.json
+ *     jobs/<job_id>/runs/<run_id>/transcript.jsonl
+ *     jobs/<job_id>/runs/<run_id>/screenshots/NN-slug.png
+ *     jobs/<job_id>/runs/<run_id>/session.webm
+ *
+ * The basenames are load-bearing: the upload presign keys are derived from
+ * the path relative to the run directory, and the server's own allowlist
+ * and lifecycle rules (the S3 lifecycle expires every `session.webm` and
+ * `transcript.jsonl` at 90 days) are written against exactly these names.
+ */
+
+export const RAVENSIGHT_DIR = '.ravensight';
+
+/** @param {string} repoRoot */
+export function ravensightDir(repoRoot) {
+  return path.join(repoRoot, RAVENSIGHT_DIR);
+}
+
+/** @param {string} repoRoot */
+export function jobsDir(repoRoot) {
+  return path.join(ravensightDir(repoRoot), 'jobs');
+}
+
+/**
+ * @param {string} repoRoot
+ * @param {string} jobId
+ */
+export function jobDir(repoRoot, jobId) {
+  return path.join(jobsDir(repoRoot), jobId);
+}
+
+/**
+ * @param {string} repoRoot
+ * @param {string} jobId
+ * @param {string} runId
+ */
+export function runDir(repoRoot, jobId, runId) {
+  return path.join(jobDir(repoRoot, jobId), 'runs', runId);
+}
+
+/** @param {string} dir a run directory */
+export function screenshotsDir(dir) {
+  return path.join(dir, 'screenshots');
+}
+
+/**
+ * The file names inside a run directory, in upload order: reports first,
+ * then screenshots, then the opt-in transcript and video (spec 17 section
+ * 1.7).
+ */
+export const RUN_FILES = Object.freeze({
+  reportMd: 'report.md',
+  reportJson: 'report.json',
+  usageJson: 'usage.json',
+  transcript: 'transcript.jsonl',
+  video: 'session.webm'
+});
+
+export const JOB_FILES = Object.freeze({
+  state: 'state.json',
+  capabilityReport: 'capability-report.json',
+  aggregateMd: 'aggregate-report.md',
+  aggregateJson: 'aggregate-report.json'
+});
+
+export default {
+  RAVENSIGHT_DIR,
+  RUN_FILES,
+  JOB_FILES,
+  ravensightDir,
+  jobsDir,
+  jobDir,
+  runDir,
+  screenshotsDir
+};