ravensight-playtest 0.2.0 → 0.2.3

package/README.md

@@ -367,7 +367,10 @@ In your repo:
 ```
 
 A `fix` job's run directory holds `changeset.json`, `report.md` (the pull
-request body) and `usage.json`, plus the screenshots it was shown as evidence.
+request body) and `usage.json`, plus the screenshots it was shown as evidence,
+and a local `transcript.jsonl` of the apply loop (which tool, on what, how
+much came back) that is never uploaded and is how an attempt that wrote
+nothing gets explained afterwards.
 
 Add `.ravensight/jobs/` to your `.gitignore`; `init` does it for you when you
 already have one.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ravensight-playtest",
-  "version": "0.2.0",
+  "version": "0.2.3",
   "description": "AI personas play your game and file reports to Ravensight.",
   "license": "MIT",
   "homepage": "https://ravensight.io/docs/#playtest",

package/src/commands/fix.js

@@ -1,5 +1,5 @@
 import { randomUUID } from 'node:crypto';
-import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
+import { appendFile, mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import { optionsFrom } from '../run/args.js';
 import { getDeps } from '../run/deps.js';
@@ -443,7 +443,13 @@ async function fix(target, flags, injected, deps, log) {
       ledger,
       log,
       onProgress: progress => {
-        if (options.verbose) log(`  ${progress.tool} ${progress.target || ''}`);
+        if (options.verbose) {
+          log(`  ${progress.tool} ${progress.target || ''} (${progress.resultChars} chars${progress.isError ? ', error' : ''}${progress.repeat ? ', repeat' : ''})`);
+        }
+        // A local transcript of the apply loop: which tools, on what, how much
+        // came back. Never uploaded; it is how a fix that wrote nothing gets
+        // explained afterwards, which the first production attempt could not be.
+        appendFile(path.join(directory, 'transcript.jsonl'), `${JSON.stringify({ at: new Date().toISOString(), ...progress })}\n`).catch(() => {});
       },
       shouldStop: () => interrupt.signalled || heartbeat.canceled
     });

package/src/dashboard.js

@@ -36,24 +36,34 @@ export function dashboardUrl(apiUrl = resolveApiUrl()) {
   return url.toString().replace(/\/+$/, '');
 }
 
+/**
+ * The dashboard's playtest routes carry no game in the path: the switcher
+ * holds the game, and a link names it in the query so the router can select
+ * it on arrival. `/games/<id>/playtest/...` was the first cut and answered
+ * "That page does not exist" for every link this CLI printed.
+ */
+function playtestPage(gameId, page, apiUrl) {
+  return `${dashboardUrl(apiUrl)}/playtest${page}?game=${encodeURIComponent(gameId)}`;
+}
+
 /** The dashboard page for one game's playtest surface. */
 export function gameLink(gameId, apiUrl) {
-  return `${dashboardUrl(apiUrl)}/games/${encodeURIComponent(gameId)}/playtest`;
+  return playtestPage(gameId, '', apiUrl);
 }
 
 /** The dashboard page for one job. Matches `links.dashboard` from POST /jobs. */
 export function jobLink(gameId, jobId, apiUrl) {
-  return `${gameLink(gameId, apiUrl)}/jobs/${encodeURIComponent(jobId)}`;
+  return playtestPage(gameId, `/jobs/${encodeURIComponent(jobId)}`, apiUrl);
 }
 
 /** The post job review form for one job. */
 export function reviewLink(gameId, jobId, apiUrl) {
-  return `${jobLink(gameId, jobId, apiUrl)}/review`;
+  return playtestPage(gameId, `/jobs/${encodeURIComponent(jobId)}/review`, apiUrl);
 }
 
 /** The brief form. */
 export function briefLink(gameId, apiUrl) {
-  return `${gameLink(gameId, apiUrl)}/brief`;
+  return playtestPage(gameId, '/brief', apiUrl);
 }
 
 /**
@@ -63,7 +73,7 @@ export function briefLink(gameId, apiUrl) {
  * server can keep true.
  */
 export function trustLink(gameId, apiUrl) {
-  return `${gameLink(gameId, apiUrl)}/what-leaves-your-machine`;
+  return playtestPage(gameId, '/what-leaves-your-machine', apiUrl);
 }
 
 /**

package/src/run/fixLoop.js

@@ -37,6 +37,32 @@ export const SUMMARY_MAX_TOKENS = 4096;
  */
 export const MAX_FIX_ACTIONS = 60;
 
+/**
+ * How many tool-answering turns the plan step may take before it has to
+ * answer. A real codebase takes several reads to locate a behavior; one
+ * repair turn (the first cut) produced empty plans on every real repository.
+ */
+export const PLAN_MAX_TURNS = 6;
+
+/**
+ * Every this-many apply turns without a write, the loop tells the model how
+ * many actions are left and asks it to write or say what blocks it. A model
+ * that is lost otherwise reads until the budget warning ends the run.
+ */
+export const NUDGE_EVERY = 8;
+
+/** The text of a tool result, whichever of the two content shapes it uses. */
+function resultText(result) {
+  if (!result) return '';
+  if (typeof result.content === 'string') return result.content;
+  if (Array.isArray(result.content)) return result.content.map(block => (block && block.text) || '').join('');
+  return '';
+}
+
+function resultLength(result) {
+  return resultText(result).length;
+}
+
 /** A plan that says nothing, which is what a malformed answer becomes. */
 export const EMPTY_PLAN = Object.freeze({ hypothesis: '', files_to_change: [], out_of_scope: [] });
 
@@ -176,7 +202,11 @@ export async function planFix({
   let repaired = false;
   let plan = null;
 
-  for (let attempt = 1; attempt <= 2 && !plan; attempt += 1) {
+  // Reading turns are answered and counted; a turn with no tool calls is
+  // the answer, and one repair is allowed when that answer is not JSON. At
+  // the ceiling the model is told to answer now, and if it still reaches for
+  // a tool the plan stays empty: planning is not where the budget belongs.
+  while (turns < PLAN_MAX_TURNS + 1 && !plan) {
     const answer = await callStepWithRetry({
       client,
       jobId,
@@ -196,9 +226,13 @@ export async function planFix({
       plan = normalisePlan(parsed);
       break;
     }
-    if (attempt === 2) break;
+    const reading = answer.toolUses.length > 0 && Boolean(repo);
+    if (!reading) {
+      if (repaired) break;
+      repaired = true;
+    }
+    if (turns > PLAN_MAX_TURNS) break;
 
-    repaired = true;
     messages.push({
       role: 'assistant',
       content: [
@@ -212,16 +246,15 @@ export async function planFix({
       const executed = await repo.execute(toolUse);
       results.push(executed.result);
     }
-    messages.push({
-      role: 'user',
-      content: [
-        ...results,
-        { type: 'text', text: 'That was not the plan object. Answer with the JSON object only, nothing before or after it.' }
-      ]
-    });
+    const nudge = turns >= PLAN_MAX_TURNS
+      ? 'No more reading. Answer now with the JSON object only, nothing before or after it.'
+      : reading
+        ? `${PLAN_MAX_TURNS - turns} more reading turn(s) before you have to answer with the JSON object.`
+        : 'That was not the plan object. Answer with the JSON object only, nothing before or after it.';
+    messages.push({ role: 'user', content: [...results, { type: 'text', text: nudge }] });
   }
 
-  if (!plan) log('The plan did not come back as JSON after one repair, so the fix proceeds without one.');
+  if (!plan) log(`The plan did not come back as JSON within ${PLAN_MAX_TURNS} reading turns and one repair, so the fix proceeds without one.`);
   return { plan: plan || { ...EMPTY_PLAN }, turns, repaired, messages };
 }
 
@@ -275,7 +308,7 @@ export async function applyFix({
     content: [{
       type: 'text',
       text: [
-        'Make the change now. Read before you write: `write_file` replaces the whole file, so send the file back with your edit in it.',
+        'Make the change now. Read before you edit. Change an existing file with `edit_file` (old_text copied verbatim, new_text in its place); use `write_file` only for a new file, because it replaces the whole file and a large file will not fit in one answer.',
         'Change as little as possible, keep the style of the code around it, and stop as soon as the fix is in. When you are done, say in one line what you changed and call no more tools.',
         '',
         `<plan>\n${JSON.stringify(plan || EMPTY_PLAN, null, 2)}\n</plan>`,
@@ -286,6 +319,10 @@ export async function applyFix({
 
   let turns = 0;
   let stop = 'max_actions';
+  // Every call the loop has already answered, so a lost model that asks the
+  // same question twice is told so instead of paying for the same answer.
+  const answered = new Map();
+  let turnsSinceWrite = 0;
   while (turns < maxActions) {
     if (shouldStop()) {
       stop = 'interrupted';
@@ -328,10 +365,32 @@ export async function applyFix({
       ]
     });
     const results = [];
+    let wroteThisTurn = false;
     for (const toolUse of answer.toolUses) {
+      const key = `${toolUse.name}:${JSON.stringify(toolUse.input || {})}`;
+      const repeat = answered.get(key);
       const executed = await repo.execute(toolUse);
-      results.push(executed.result);
-      onProgress({ turn: turns, tool: toolUse.name, target: toolUse.input && (toolUse.input.path || toolUse.input.pattern) });
+      const result = repeat === undefined
+        ? executed.result
+        : { ...executed.result, content: [{ type: 'text', text: `(identical to a call you already made on turn ${repeat}; the answer has not changed)\n${resultText(executed.result)}` }] };
+      answered.set(key, repeat === undefined ? turns : repeat);
+      results.push(result);
+      if ((toolUse.name === 'write_file' || toolUse.name === 'edit_file') && !executed.result.is_error) wroteThisTurn = true;
+      onProgress({
+        turn: turns,
+        tool: toolUse.name,
+        target: toolUse.input && (toolUse.input.path || toolUse.input.pattern),
+        resultChars: resultLength(executed.result),
+        isError: Boolean(executed.result.is_error),
+        repeat: repeat !== undefined
+      });
+    }
+    turnsSinceWrite = wroteThisTurn ? 0 : turnsSinceWrite + 1;
+    if (turnsSinceWrite > 0 && turnsSinceWrite % NUDGE_EVERY === 0 && turns < maxActions) {
+      results.push({
+        type: 'text',
+        text: `You have used ${turns} of ${maxActions} actions and written nothing. Either make the change now with edit_file (or write_file for a new file), or answer in one line, with no tool call, why it cannot be made.`
+      });
     }
     messages.push({ role: 'user', content: results });
 

package/src/run/repoTools.js

@@ -68,7 +68,13 @@ export const MAX_READ_BYTES = 120000;
 export const MAX_GLOB_RESULTS = 400;
 
 /** How many matching lines a grep answers with. */
-export const MAX_GREP_RESULTS = 200;
+/**
+ * Sixty, not two hundred: at three hundred characters a line the old cap
+ * let one grep put 60 KB into the context, and forty greps in one apply loop
+ * ate the whole model budget without a write. A capped answer says so, so
+ * the model narrows rather than repeats.
+ */
+export const MAX_GREP_RESULTS = 60;
 
 /** The tool definitions for `game_profile.triage`. */
 export const PROFILE_TOOLS = Object.freeze([
@@ -126,10 +132,34 @@ export const MAX_WRITE_BYTES = 200000;
 /** How many distinct paths one run may write. A fix is a patch, not a rewrite. */
 export const MAX_WRITES_PER_RUN = 20;
 
-/** The one write tool, defined here so nothing else can invent a second one. */
+/**
+ * The two writing tools, defined here so nothing else can invent a third.
+ *
+ * `edit_file` exists because whole-file writes do not survive contact with a
+ * real codebase: the first production fix attempt read an 84 KB C++ file and
+ * then could not send it back inside the step's output ceiling, so its one
+ * write arrived as malformed input and the attempt ended with nothing on the
+ * branch. An exact-text replacement is a few hundred bytes for the same edit.
+ */
+export const EDIT_FILE_TOOL = Object.freeze({
+  name: 'edit_file',
+  description: 'Replace one exact passage of a text file in the repository with new text. Paths are relative to the repository root. old_text must appear exactly once in the file (or set replace_all for every occurrence); copy it verbatim from read_file, whitespace included. This is the tool for changing a file that already exists.',
+  input_schema: {
+    type: 'object',
+    additionalProperties: false,
+    required: ['path', 'old_text', 'new_text'],
+    properties: {
+      path: { type: 'string' },
+      old_text: { type: 'string' },
+      new_text: { type: 'string' },
+      replace_all: { type: 'boolean' }
+    }
+  }
+});
+
 export const WRITE_FILE_TOOL = Object.freeze({
   name: 'write_file',
-  description: 'Write a text file in the repository, replacing it whole. Paths are relative to the repository root. Read the file first: there is no patch tool, so the content you send is the file.',
+  description: 'Write a text file in the repository, replacing it whole. Paths are relative to the repository root. Use it for a new file or a small one; for a change to an existing file use edit_file, because the content you send here is the whole file.',
   input_schema: {
     type: 'object',
     additionalProperties: false,
@@ -141,8 +171,8 @@ export const WRITE_FILE_TOOL = Object.freeze({
   }
 });
 
-/** The tool definitions for `fix.apply`: the read-only three, plus the write. */
-export const WRITE_TOOLS = Object.freeze([...PROFILE_TOOLS, WRITE_FILE_TOOL]);
+/** The tool definitions for `fix.apply`: the read-only three, plus the two writes. */
+export const WRITE_TOOLS = Object.freeze([...PROFILE_TOOLS, EDIT_FILE_TOOL, WRITE_FILE_TOOL]);
 
 /**
  * Compile a glob into a regular expression. Supports `**`, `*` and `?`,
@@ -388,6 +418,14 @@ export function createRepoTools({ root, tools = PROFILE_TOOLS }) {
           const left = MAX_WRITES_PER_RUN - written.size;
           return { result: ok(toolUse.id, `Wrote ${planned.relative} (${planned.bytes} bytes). ${left} more file(s) may be written in this run.`) };
         }
+        if (toolUse.name === 'edit_file' && canWrite) {
+          const planned = await planEdit(root, input, written);
+          if (planned.refusal) return { result: ok(toolUse.id, planned.refusal, true) };
+          await doWrite(planned.absolute, planned.content);
+          written.add(planned.relative);
+          const left = MAX_WRITES_PER_RUN - written.size;
+          return { result: ok(toolUse.id, `Edited ${planned.relative}: ${planned.replaced} occurrence(s) replaced, now ${planned.bytes} bytes. ${left} more file(s) may be written in this run.`) };
+        }
         if (toolUse.name === 'glob') {
           const matcher = globToRegExp(input.pattern);
           const matched = (await files()).filter(file => matcher.test(file)).slice(0, MAX_GLOB_RESULTS);
@@ -495,6 +533,54 @@ export async function planWrite(root, input, written = new Set()) {
  * @param {string} absolute
  * @param {string} content
  */
+/**
+ * Everything `planWrite` checks about the path, then the replacement itself:
+ * the file has to exist and be text, `old_text` has to be found exactly once
+ * unless `replace_all`, and the result has to fit the same byte ceiling a
+ * whole write does. Refusals name the count found, because "not found" and
+ * "found three times" call for different next moves.
+ *
+ * @param {string} root
+ * @param {Object} input
+ * @param {Set<string>} written
+ * @returns {Promise<{refusal: string}|{relative: string, absolute: string, content: string, bytes: number, replaced: number}>}
+ */
+export async function planEdit(root, input, written = new Set()) {
+  const oldText = input && input.old_text;
+  const newText = input && input.new_text;
+  if (typeof oldText !== 'string' || oldText.length === 0) {
+    return { refusal: 'Refused: edit_file needs old_text, the exact passage to replace, copied verbatim from read_file.' };
+  }
+  if (typeof newText !== 'string') return { refusal: 'Refused: new_text has to be a string (it may be empty to delete the passage).' };
+  // The path rules are the write rules; the placeholder content only has to be
+  // a string so the shared checks run.
+  const planned = await planWrite(root, { path: input && input.path, content: '' }, written);
+  if (planned.refusal) return { refusal: planned.refusal.replace('write_file', 'edit_file') };
+
+  let current;
+  try {
+    current = await readFile(planned.absolute, 'utf8');
+  } catch {
+    return { refusal: `Refused: ${planned.relative} does not exist yet. Use write_file to create a new file.` };
+  }
+  if (current.includes(NUL)) return { refusal: `Refused: ${planned.relative} is not a text file.` };
+
+  const occurrences = current.split(oldText).length - 1;
+  if (occurrences === 0) {
+    return { refusal: `Refused: old_text was not found in ${planned.relative}. Read the file again and copy the passage exactly, whitespace included.` };
+  }
+  if (occurrences > 1 && !(input && input.replace_all)) {
+    return { refusal: `Refused: old_text appears ${occurrences} times in ${planned.relative}. Include more surrounding lines so it is unique, or set replace_all.` };
+  }
+  const content = input && input.replace_all ? current.split(oldText).join(newText) : current.replace(oldText, () => newText);
+  if (content.includes(NUL)) return { refusal: 'Refused: new_text holds a NUL byte, so that is not text.' };
+  const bytes = Buffer.byteLength(content, 'utf8');
+  if (bytes > MAX_WRITE_BYTES) {
+    return { refusal: `Refused: ${planned.relative} would be ${bytes} bytes and one write may be at most ${MAX_WRITE_BYTES}.` };
+  }
+  return { relative: planned.relative, absolute: planned.absolute, content, bytes, replaced: input && input.replace_all ? occurrences : 1 };
+}
+
 async function doWrite(absolute, content) {
   await mkdir(path.dirname(absolute), { recursive: true });
   const tmp = `${absolute}.${process.pid}.${randomUUID()}.tmp`;
@@ -532,7 +618,11 @@ async function doGrep(root, all, input) {
       if (regex.test(lines[i])) hits.push(`${relative}:${i + 1}: ${lines[i].trim().slice(0, 300)}`);
     }
   }
-  return hits.length > 0 ? hits.join('\n') : '(no matches)';
+  if (hits.length === 0) return '(no matches)';
+  const capped = hits.length >= MAX_GREP_RESULTS;
+  return capped
+    ? `${hits.join('\n')}\n(stopped at ${MAX_GREP_RESULTS} matches; narrow the pattern or add a glob)`
+    : hits.join('\n');
 }
 
 function ok(id, text, isError = false) {
@@ -544,6 +634,8 @@ export default {
   PROFILE_TOOLS,
   READ_ONLY_TOOLS,
   WRITE_TOOLS,
+  EDIT_FILE_TOOL,
+  planEdit,
   globToRegExp,
   listFiles,
   planWrite,