ravensight-playtest 0.1.0 → 0.2.0

package/src/commands/run.js

@@ -586,8 +586,12 @@ export async function playPersonaRun({
  *
  * A refused delivery does not take the job down. `finish` still settles it, the
  * artifacts are on disk, and `resume` can retry.
+ *
+ * Exported for `fix`, whose verify run is an ordinary persona run and has to be
+ * uploaded and completed in exactly this order, including the `uploading`
+ * transition the server's table allows from `reporting` and nowhere else.
  */
-async function deliver({ deps, api, gameId, jobId, runId, directory, outcome, options, log, journal, repoRoot }) {
+export async function deliver({ deps, api, gameId, jobId, runId, directory, outcome, options, log, journal, repoRoot }) {
   const body = {
     state: outcome.state,
     reason: outcome.quitDetail || '',
@@ -807,9 +811,14 @@ async function publishAggregate({ deps, api, gameId, jobId, repoRoot, directory,
  * the default behaviour: someone pressing Ctrl-C twice means it now, and
  * holding the process open to be tidy would be the wrong answer.
  *
+ * `message` is for a command whose work a resume cannot pick up. `run` and
+ * `resume` can both be carried on, and say so; `fix` cannot, because its runs
+ * are half-applied edits rather than a replayable playthrough, and promising a
+ * resume there would send the developer to a command that would refuse them.
+ *
  * @returns {{signalled: boolean, settle: () => Promise<void>, release: () => void}}
  */
-export function createInterruptHandler({ api, gameId, jobId, repoRoot, journal, inFlight, log, injected = {} }) {
+export function createInterruptHandler({ api, gameId, jobId, repoRoot, journal, inFlight, log, message = null, injected = {} }) {
   const signals = injected.signals || ['SIGINT', 'SIGTERM'];
   const target = injected.process || process;
   const state = { signalled: false, signal: null };
@@ -818,7 +827,7 @@ export function createInterruptHandler({ api, gameId, jobId, repoRoot, journal,
     if (state.signalled) return;
     state.signalled = true;
     state.signal = signal;
-    log(`\n${signal}: finishing the turn in flight, then stopping. Nothing is lost; "ravensight-playtest resume ${jobId}" carries on.`);
+    log(`\n${signal}: ${message || `finishing the turn in flight, then stopping. Nothing is lost; "ravensight-playtest resume ${jobId}" carries on.`}`);
     // Hand the signal back to the default behaviour, so a second press kills
     // the process rather than being swallowed by a handler that is now done.
     for (const name of signals) target.removeListener(name, onSignal);

package/src/packs/index.js

@@ -117,10 +117,25 @@ export async function sync({ api, gameId, modules }) {
  * An authentication or authorization failure is never treated as "offline":
  * falling back to a cache on a 401 would let a revoked token keep working.
  *
- * @param {{api: Object, gameId?: string, modules?: string[], offline?: boolean}} options
+ * A `jobId` asks for the bundle in one job's context, which is the only way
+ * `graph_context` is populated. That answer is about one job and can never
+ * serve another, so it is fetched straight through: caching it would put a file
+ * per job in the cache directory that nothing would ever read again.
+ *
+ * @param {{api: Object, gameId?: string, modules?: string[], jobId?: string, offline?: boolean}} options
  * @returns {Promise<PackResult>}
  */
-export async function get({ api, gameId, modules, offline = false }) {
+export async function get({ api, gameId, modules, jobId = null, offline = false }) {
+  if (jobId) {
+    const answer = await api.packs.bundle({ gameId, modules, jobId });
+    return {
+      bundle: answer.body,
+      etag: answer.etag || null,
+      fromCache: false,
+      fetchedAt: new Date().toISOString(),
+      stale: false
+    };
+  }
   const key = { apiUrl: api.apiUrl, gameId, modules };
   const existing = await cached(key);
 

package/src/run/args.js

@@ -97,7 +97,12 @@ export function optionsFrom(flags = {}) {
         idempotencyKey: flags.idempotencyKey,
         // `resume` declares this one in commander rather than leaving it to the
         // argv leftovers, so it arrives as a flag and has to be read here too.
-        restartInterrupted: flags.restartInterrupted
+        restartInterrupted: flags.restartInterrupted,
+        // `fix` declares these three in commander for the same reason: they are
+        // about the branch and the verify, not about driving a game.
+        branch: flags.branch,
+        allowDirty: flags.allowDirty,
+        verify: flags.verify
       })
     },
     unknown

package/src/run/fixLoop.js

@@ -0,0 +1,430 @@
+import { parseJsonObject } from './json.js';
+import { callStepWithRetry, ProxyError, STEPS } from './model.js';
+import { createRepoTools, WRITE_TOOLS } from './repoTools.js';
+
+/**
+ * The `fix` module's three model steps: plan, apply, summarise.
+ *
+ * They are separate steps rather than one loop because they are priced and
+ * tooled differently, and because the boundaries are where the safety is:
+ *
+ * - `fix.plan` reads the finding and answers a plan. One call, plus at most
+ *   one repair turn when the answer is not JSON. It cannot write, so a plan
+ *   that is wrong costs a turn rather than a file.
+ * - `fix.apply` is the loop, and the only step anywhere in this CLI whose
+ *   tools include `write_file`. Its ceiling is the action budget, and its
+ *   guards are in ./repoTools.js rather than in the prompt.
+ * - `fix.summarize` has no tools at all. It reads the diff that was actually
+ *   produced and writes the pull request text, so the summary describes the
+ *   change on disk instead of the change that was planned.
+ *
+ * What none of them can do: run a command, reach the network, read a
+ * credential file, write outside the repository, or publish anything. The
+ * developer's own review is the gate between this and a pull request, which
+ * is why the command ends by printing a command rather than running one.
+ */
+
+/** Tokens asked for per step. The proxy clamps each to the step's own ceiling. */
+export const PLAN_MAX_TOKENS = 2048;
+/** Higher than the others: a `write_file` call carries a whole file. */
+export const APPLY_MAX_TOKENS = 8192;
+export const SUMMARY_MAX_TOKENS = 4096;
+
+/**
+ * The hard ceiling on apply turns, whatever the pack or the developer asks
+ * for. A fix that has not landed in sixty turns is not one turn away, and
+ * every turn is billable.
+ */
+export const MAX_FIX_ACTIONS = 60;
+
+/** A plan that says nothing, which is what a malformed answer becomes. */
+export const EMPTY_PLAN = Object.freeze({ hypothesis: '', files_to_change: [], out_of_scope: [] });
+
+/**
+ * The plan step's own image ceiling. The proxy counts image blocks across the
+ * whole `messages` array and answers 400 `too_many_images` past it, so this
+ * is a ceiling on the context rather than on one turn.
+ */
+export const MAX_PLAN_IMAGES = 2;
+
+/**
+ * The action budget for `fix.apply`: the smallest of what the developer
+ * asked for and what routing allows, and never more than `MAX_FIX_ACTIONS`.
+ *
+ * @param {Object} options
+ * @param {Object} pack
+ * @returns {number}
+ */
+export function resolveFixMaxActions(options = {}, pack = {}) {
+  const candidates = [
+    Number.isInteger(options.maxActions) ? options.maxActions : null,
+    Number(pack && pack.routing && pack.routing.max_actions) || null
+  ].filter(value => Number.isFinite(value) && value > 0);
+  const asked = candidates.length > 0 ? Math.min(...candidates) : MAX_FIX_ACTIONS;
+  return Math.max(1, Math.min(asked, MAX_FIX_ACTIONS));
+}
+
+/**
+ * The system prompt every fix step shares: the pack's `fix` skill, then the
+ * facts about this repository and this attempt.
+ *
+ * @param {Object} args
+ * @returns {Array<Object>}
+ */
+export function fixSystem({ skill, jobId, branch, baseCommit, engine = null }) {
+  return [
+    { type: 'text', text: skill },
+    {
+      type: 'text',
+      text: [
+        `job_id: ${jobId}`,
+        'the paths you read and write are relative to the repository root; never absolute',
+        `engine recorded on this game: ${engine || 'unknown'}`,
+        `branch: ${branch} (created for this attempt, from ${baseCommit || 'unknown'})`,
+        'you cannot run commands, reach the network, or commit anything. The developer reviews the branch and opens the pull request themselves.'
+      ].join('\n'),
+      cache_control: { type: 'ephemeral' }
+    }
+  ];
+}
+
+/**
+ * The bundle's map of the codebase, as content blocks.
+ *
+ * It goes in the user message of the two steps that read code rather than in
+ * the shared system block, so neither step pays for the other's copy and the
+ * summarise step, which reads no code at all, pays for none of it.
+ *
+ * @param {Object|null} codeBrief
+ * @param {Object|null} graphContext
+ * @returns {Array<string>} lines
+ */
+export function bundleLines(codeBrief, graphContext) {
+  const lines = [];
+  if (codeBrief) lines.push(`<code_brief>\n${JSON.stringify(codeBrief).slice(0, 4000)}\n</code_brief>`);
+  if (graphContext) lines.push(`<graph_context>\n${JSON.stringify(graphContext).slice(0, 4000)}\n</graph_context>`);
+  return lines;
+}
+
+/**
+ * The user message the plan step reads: what was found, how to see it, what
+ * the persona said, up to two screenshots, and what is already known to be
+ * intended.
+ *
+ * @param {Object} args
+ * @returns {Array<Object>} content blocks
+ */
+export function planContent({
+  finding,
+  reproSteps = [],
+  personaQuote = null,
+  images = [],
+  intentNotes = [],
+  reportMarkdown = null,
+  codeBrief = null,
+  graphContext = null
+}) {
+  const lines = [
+    'A persona filed this finding while playing the game. Work out why it happens and plan the smallest change that fixes it.',
+    '',
+    `<finding>\n${JSON.stringify(finding, null, 2)}\n</finding>`,
+    `<repro_steps>\n${reproSteps.length > 0 ? reproSteps.map((step, index) => `${index + 1}. ${step}`).join('\n') : 'none recorded'}\n</repro_steps>`,
+    `<persona_quote>\n${personaQuote || 'none recorded'}\n</persona_quote>`,
+    `<design_intent_notes>\n${intentNotes.length > 0 ? intentNotes.map(note => `- ${note}`).join('\n') : 'none'}\n</design_intent_notes>`,
+    ...bundleLines(codeBrief, graphContext)
+  ];
+  if (reportMarkdown) lines.push(`<run_report>\n${String(reportMarkdown).slice(0, 12000)}\n</run_report>`);
+  lines.push(
+    '',
+    'Answer with this JSON object and nothing else:',
+    '{ "hypothesis": "one or two sentences on the cause", "files_to_change": ["repo/relative/path"], "out_of_scope": ["what you are deliberately not touching"] }'
+  );
+  return [
+    { type: 'text', text: lines.join('\n') },
+    ...images.slice(0, MAX_PLAN_IMAGES).map(image => ({
+      type: 'image',
+      source: { type: 'base64', media_type: image.mediaType || 'image/png', data: image.base64 }
+    }))
+  ];
+}
+
+/**
+ * Plan the fix: one call, and one repair turn when the answer is not a JSON
+ * object.
+ *
+ * The step's tool allowlist is the read-only three, so the tools are offered
+ * and a model that would rather read a file first can. Tool calls are
+ * answered properly in the repair turn, which keeps the message sequence
+ * legal and gets the reading it asked for; it is still one repair, never a
+ * loop, because planning is not where the budget belongs.
+ *
+ * @param {Object} args
+ * @returns {Promise<{plan: Object, turns: number, repaired: boolean, messages: Array}>}
+ */
+export async function planFix({
+  client,
+  jobId,
+  runId,
+  system,
+  content,
+  repo,
+  ledger = null,
+  log = () => {}
+}) {
+  const messages = [{ role: 'user', content }];
+  let turns = 0;
+  let repaired = false;
+  let plan = null;
+
+  for (let attempt = 1; attempt <= 2 && !plan; attempt += 1) {
+    const answer = await callStepWithRetry({
+      client,
+      jobId,
+      runId,
+      step: STEPS.fixPlan,
+      system,
+      messages,
+      tools: repo ? repo.tools : undefined,
+      maxTokens: PLAN_MAX_TOKENS
+    });
+    turns += 1;
+    if (ledger) ledger.record({ model: answer.model, usage: answer.usage });
+    if (answer.budget === 'warn') log('budget warning: this fix has used at least 80 percent of its model budget.');
+
+    const parsed = parseJsonObject(answer.text);
+    if (parsed) {
+      plan = normalisePlan(parsed);
+      break;
+    }
+    if (attempt === 2) break;
+
+    repaired = true;
+    messages.push({
+      role: 'assistant',
+      content: [
+        ...(answer.text.trim() ? [{ type: 'text', text: answer.text }] : []),
+        ...answer.toolUses.map(toolUse => ({ type: 'tool_use', id: toolUse.id, name: toolUse.name, input: toolUse.malformed ? {} : toolUse.input }))
+      ]
+    });
+    const results = [];
+    for (const toolUse of answer.toolUses) {
+      if (!repo) break;
+      const executed = await repo.execute(toolUse);
+      results.push(executed.result);
+    }
+    messages.push({
+      role: 'user',
+      content: [
+        ...results,
+        { type: 'text', text: 'That was not the plan object. Answer with the JSON object only, nothing before or after it.' }
+      ]
+    });
+  }
+
+  if (!plan) log('The plan did not come back as JSON after one repair, so the fix proceeds without one.');
+  return { plan: plan || { ...EMPTY_PLAN }, turns, repaired, messages };
+}
+
+/**
+ * @param {Object} parsed
+ * @returns {{hypothesis: string, files_to_change: Array<string>, out_of_scope: Array<string>}}
+ */
+export function normalisePlan(parsed) {
+  const paths = Array.isArray(parsed.files_to_change) ? parsed.files_to_change : [];
+  return {
+    hypothesis: typeof parsed.hypothesis === 'string' ? parsed.hypothesis : '',
+    files_to_change: paths
+      .map(entry => (typeof entry === 'string' ? entry : entry && entry.path))
+      .filter(entry => typeof entry === 'string' && entry.trim())
+      .map(entry => entry.trim()),
+    out_of_scope: (Array.isArray(parsed.out_of_scope) ? parsed.out_of_scope : [])
+      .map(entry => (typeof entry === 'string' ? entry : JSON.stringify(entry)))
+      .filter(Boolean)
+  };
+}
+
+/**
+ * The apply loop: read, write, repeat, until the model says it is done, the
+ * action budget runs out, or the run's model budget does.
+ *
+ * A 402 is not an error here. The files already written are on the branch and
+ * are what the developer will review, so the loop stops, says so, and the
+ * command carries on to report what exists.
+ *
+ * @param {Object} args
+ * @returns {Promise<{turns: number, written: Array<string>, stop: string}>}
+ */
+export async function applyFix({
+  client,
+  jobId,
+  runId,
+  repoRoot,
+  system,
+  plan,
+  maxActions,
+  codeBrief = null,
+  graphContext = null,
+  repo = createRepoTools({ root: repoRoot, tools: WRITE_TOOLS }),
+  ledger = null,
+  log = () => {},
+  onProgress = () => {},
+  shouldStop = () => false
+}) {
+  const messages = [{
+    role: 'user',
+    content: [{
+      type: 'text',
+      text: [
+        'Make the change now. Read before you write: `write_file` replaces the whole file, so send the file back with your edit in it.',
+        'Change as little as possible, keep the style of the code around it, and stop as soon as the fix is in. When you are done, say in one line what you changed and call no more tools.',
+        '',
+        `<plan>\n${JSON.stringify(plan || EMPTY_PLAN, null, 2)}\n</plan>`,
+        ...bundleLines(codeBrief, graphContext)
+      ].join('\n')
+    }]
+  }];
+
+  let turns = 0;
+  let stop = 'max_actions';
+  while (turns < maxActions) {
+    if (shouldStop()) {
+      stop = 'interrupted';
+      break;
+    }
+    turns += 1;
+    let answer;
+    try {
+      answer = await callStepWithRetry({
+        client,
+        jobId,
+        runId,
+        step: STEPS.fixApply,
+        system,
+        messages,
+        tools: repo.tools,
+        maxTokens: APPLY_MAX_TOKENS
+      });
+    } catch (error) {
+      if (error instanceof ProxyError && error.isBudget) {
+        log('This fix has spent its model budget. Whatever was written is on the branch.');
+        stop = 'budget';
+        break;
+      }
+      throw error;
+    }
+    if (ledger) ledger.record({ model: answer.model, usage: answer.usage });
+
+    if (answer.toolUses.length === 0) {
+      messages.push({ role: 'assistant', content: [{ type: 'text', text: answer.text || '(done)' }] });
+      stop = 'end_turn';
+      break;
+    }
+
+    messages.push({
+      role: 'assistant',
+      content: [
+        ...(answer.text.trim() ? [{ type: 'text', text: answer.text }] : []),
+        ...answer.toolUses.map(toolUse => ({ type: 'tool_use', id: toolUse.id, name: toolUse.name, input: toolUse.malformed ? {} : toolUse.input }))
+      ]
+    });
+    const results = [];
+    for (const toolUse of answer.toolUses) {
+      const executed = await repo.execute(toolUse);
+      results.push(executed.result);
+      onProgress({ turn: turns, tool: toolUse.name, target: toolUse.input && (toolUse.input.path || toolUse.input.pattern) });
+    }
+    messages.push({ role: 'user', content: results });
+
+    if (answer.budget === 'warn') {
+      log('budget warning: this fix has used at least 80 percent of its model budget. Wrapping up.');
+      stop = 'budget_warning';
+      break;
+    }
+  }
+
+  return { turns, written: repo.written, stop };
+}
+
+/**
+ * The pull request text, written from the diff that exists rather than from
+ * the plan that was made.
+ *
+ * `fallback` is used when the step cannot answer at all (no budget left, or
+ * an answer with no JSON in it). A fix that landed on disk still has to be
+ * reportable, and a locally written summary that says so is more honest than
+ * failing the whole attempt over its prose.
+ *
+ * @param {Object} args
+ * @returns {Promise<{prTitle: string, prBody: string, source: string}>}
+ */
+export async function summariseFix({
+  client,
+  jobId,
+  runId,
+  system,
+  finding,
+  plan,
+  files,
+  verify,
+  branch,
+  fallback,
+  ledger = null,
+  log = () => {}
+}) {
+  const content = [{
+    type: 'text',
+    text: [
+      'Write the pull request text for the change you just made. The reviewer is the developer who owns this code and has not seen the finding.',
+      '',
+      `<finding>\n${JSON.stringify(finding, null, 2)}\n</finding>`,
+      `<plan>\n${JSON.stringify(plan || EMPTY_PLAN, null, 2)}\n</plan>`,
+      `<diff_stat>\n${files.map(file => `${file.path}: +${file.additions} -${file.deletions}`).join('\n') || 'nothing'}\n</diff_stat>`,
+      `<verify>\n${verify ? JSON.stringify(verify) : 'the fix was not verified by replaying the finding'}\n</verify>`,
+      `<branch>${branch}</branch>`,
+      '',
+      'Answer with this JSON object and nothing else:',
+      '{ "pr_title": "one line, at most 120 characters", "pr_body": "markdown: what was wrong, what changed, what the verify run showed, and what the reviewer should check by hand" }'
+    ].join('\n')
+  }];
+
+  let answer;
+  try {
+    answer = await callStepWithRetry({
+      client,
+      jobId,
+      runId,
+      step: STEPS.fixSummarize,
+      system,
+      messages: [{ role: 'user', content }],
+      maxTokens: SUMMARY_MAX_TOKENS
+    });
+  } catch (error) {
+    if (error instanceof ProxyError && error.isBudget) {
+      log('There was no budget left to write the pull request text, so it was written locally instead.');
+      return { ...fallback, source: 'local' };
+    }
+    throw error;
+  }
+  if (ledger) ledger.record({ model: answer.model, usage: answer.usage });
+
+  const parsed = parseJsonObject(answer.text) || {};
+  const title = typeof parsed.pr_title === 'string' && parsed.pr_title.trim() ? parsed.pr_title.trim() : fallback.prTitle;
+  const body = typeof parsed.pr_body === 'string' && parsed.pr_body.trim() ? parsed.pr_body : fallback.prBody;
+  return {
+    prTitle: clamp(title, 120),
+    prBody: clamp(body, 20000),
+    source: parsed.pr_title || parsed.pr_body ? 'model' : 'local'
+  };
+}
+
+/**
+ * @param {string} value
+ * @param {number} limit
+ * @returns {string}
+ */
+export function clamp(value, limit) {
+  const text = String(value || '');
+  return text.length <= limit ? text : text.slice(0, limit);
+}
+
+export default { planFix, applyFix, summariseFix, resolveFixMaxActions, fixSystem, planContent, normalisePlan };