ravensight-playtest 0.2.3 → 0.2.5

package/src/run/personaLoop.js

@@ -1,12 +1,14 @@
+import { selectAction, JEV_CONTROLLER_PROMPT } from './actionSelector.js';
 import { mkdir, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import { prepareMessages, MAX_IMAGES_IN_CONTEXT, KEEP_OBSERVATIONS, shrinkImages } from './context.js';
 import { observationHash } from './drivers/driver.js';
+import { withRollingCacheBreakpoint } from './fixLoop.js';
 import { callStepWithRetry, ProxyError, STEPS } from './model.js';
 import { RUN_FILES } from './paths.js';
 import { checkEvidence, createReportValidator, lintMarkdownSections, repairInstruction, REQUIRED_MARKDOWN_SECTIONS, stampReport, writeReportFiles } from './report.js';
 import { assertNoSecrets } from './secretScan.js';
-import { ACTION_TOOLS, PLAY_TOOLS, REPORT_TOOLS, executeTool } from './tools.js';
+import { ACTION_TOOLS, PLAY_TOOLS, SWIPE_TOOL, REPORT_TOOLS, executeTool } from './tools.js';
 import { createTranscript } from './transcript.js';
 import { createUsageLedger, writeUsageFile } from './usage.js';
 
@@ -31,11 +33,30 @@ import { createUsageLedger, writeUsageFile } from './usage.js';
  *   up, so the report (which is the product) gets written while there is
  *   still budget to write it with.
  * - A 402 means the money is gone. Nothing further is asked of the model,
- *   including the report: a partial `report.md` is written locally from the
- *   notes, `complete` is sent with `state: budget_exceeded` and no
- *   `report`, and the process exits 2.
+ *   including the report: a partial `report.md` and `report.json` are
+ *   written locally from the notes (`partial: true`, each note an
+ *   unreviewed finding), `complete` is sent with `state: budget_exceeded`
+ *   and that report, and the process exits 2. The notes are the one thing
+ *   the run bought before the money ran out, so they are not thrown away.
  */
 
+/**
+ * Order across notes and screenshots, so a partial report can attach the
+ * screenshot nearest to each note. Module wide and only ever compared within
+ * one run, where it is monotonic.
+ */
+let ORDER = 0;
+const nextOrder = () => {
+  ORDER += 1;
+  return ORDER;
+};
+
+/** `Failed to load resource: ... status of 4xx`: a missing asset or a refused call, not a crash. */
+const RESOURCE_4XX = /Failed to load resource: the server responded with a status of 4\d\d/;
+
+/** A persona note kind as a report finding category. */
+export const NOTE_CATEGORY = Object.freeze({ bug: 'bug', confusing: 'ux', suggestion: 'ux', good: 'ux', reaction: 'ux' });
+
 /** Stop injecting the "nothing changed" hint after this many identical screens. */
 export const STUCK_HINT_AFTER = 5;
 
@@ -88,7 +109,9 @@ export async function runPersona(args) {
     schemas,
     maxActions,
     maxImages = MAX_IMAGES_IN_CONTEXT,
-    keepObservations = KEEP_OBSERVATIONS,
+    // A driver may say how many observations it needs in context: a web
+    // snapshot is big and re-sent every turn, a Godot scene summary is not.
+    keepObservations = (driver && Number.isInteger(driver.keepObservations) ? driver.keepObservations : KEEP_OBSERVATIONS),
     heartbeat = null,
     onProgress = () => {},
     log = () => {},
@@ -114,6 +137,7 @@ export async function runPersona(args) {
     job_id: jobId,
     persona: persona.slug,
     driver: driverName,
+    controller: args.controller || 'standard',
     target,
     prompt_version: promptVersionOf(skill),
     brief_id: brief ? brief.brief_id || null : null,
@@ -186,7 +210,15 @@ export async function runPersona(args) {
       outcome.quitReason = 'cost_cap';
       // Nothing more is asked of the model: there is no budget left to ask
       // with, and a refused call would just be another 402.
-      await writePartialMarkdown({ runDirectory, persona, driverName, target, notes, screenshots, play, reason: 'The run reached its model budget before the report step. What the persona saw is below; there is no report.json for this run.' });
+      // A report that cannot be written (a secret shaped note, a full disk)
+      // must not turn an out of budget run into a failed one.
+      try {
+        await writePartialReport({ outcome, runDirectory, persona, driverName, target, notes, screenshots, play, skill, ledger, runId, jobId, startedAt, endedAt });
+      } catch (writeError) {
+        outcome.report = null;
+        outcome.reportMarkdown = null;
+        log(`the partial report could not be written: ${String(writeError && writeError.message)}`);
+      }
       return finish(outcome, ledger, runDirectory, transcript, heartbeat);
     }
 
@@ -228,6 +260,14 @@ export async function runPersona(args) {
     if (error instanceof ProxyError && error.isBudget) {
       outcome.state = 'budget_exceeded';
       outcome.quitReason = 'cost_cap';
+      // The money ran out in the report step itself: the notes are still
+      // worth keeping, by the same rule as a 402 during play.
+      try {
+        const play = { actionsTaken: outcome.actionsTaken, quitReason: 'cost_cap', quitDetail: String(error.message || '') };
+        await writePartialReport({ outcome, runDirectory, persona, driverName, target, notes, screenshots, play, skill, ledger, runId, jobId, startedAt, endedAt: new Date().toISOString() });
+      } catch (writeError) {
+        log(`the partial report could not be written: ${String(writeError && writeError.message)}`);
+      }
     } else if (error instanceof ProxyError && error.isClosed) {
       outcome.state = 'canceled';
       outcome.quitReason = 'error';
@@ -248,6 +288,8 @@ export async function runPersona(args) {
  * @returns {Promise<Object>}
  */
 async function playLoop({
+  api,
+  controller = 'standard',
   client,
   jobId,
   runId,
@@ -270,6 +312,25 @@ async function playLoop({
   log = () => {}
 }) {
   const system = buildSystem({ skill, persona, brief, intentNotes, driverName, target, maxActions });
+  if (controller === 'jev') system.push({ type: 'text', text: JEV_CONTROLLER_PROMPT });
+  const availableTools = driver.supportedActions
+    ? [...PLAY_TOOLS, ...(driver.supportedActions.includes('swipe') ? [SWIPE_TOOL] : [])].filter(tool => !ACTION_TOOLS.includes(tool.name) || driver.supportedActions.includes(tool.name))
+    : PLAY_TOOLS;
+  const playTools = controller === 'jev' ? availableTools.map(tool => ({
+    ...tool,
+    input_schema: {
+      ...tool.input_schema,
+      required: [...(tool.input_schema.required || []), 'visible_state', ...(tool.name === 'click' ? ['target'] : [])],
+      properties: {
+        ...tool.input_schema.properties,
+        visible_state: { type: 'string', description: 'Describe what is visibly on the current screenshot, including this action target and its coordinates. This observation is required even when the assistant text is empty. Do not claim unobserved state.' },
+        ...(tool.name === 'click' ? { target: { type: 'string', description: 'Label of the visible control. For canvas also supply x and y; the driver uses those coordinates.' } } : {})
+      }
+    }
+  })) : availableTools;
+  const recentActions = [];
+  let latestObservation = '';
+  let visualObservation = false;
   const messages = [];
   let actions = 0;
   let turns = 0;
@@ -283,6 +344,8 @@ async function playLoop({
   const maxTurns = maxActions * 3 + 20;
 
   const first = await driver.observe();
+  latestObservation = first.text;
+  visualObservation = Boolean(first.screenshotBase64);
   transcript.write({ kind: 'observation', step: first.step, text: first.text, url: first.url });
   const landing = await safeScreenshot(driver, 'landing', screenshots, transcript);
   messages.push({
@@ -317,7 +380,7 @@ async function playLoop({
         step: STEPS.personaAct,
         system,
         messages,
-        tools: PLAY_TOOLS,
+        tools: playTools,
         maxTokens: ACT_MAX_TOKENS,
         escalate: shouldEscalate(persona),
         imageBudget,
@@ -370,9 +433,23 @@ async function playLoop({
       break;
     }
 
+    let selection = null;
+    if (controller === 'jev') {
+      try {
+        selection = await selectAction({ api, jobId, runId, persona, observation: `${visualObservation ? '' : latestObservation}\nScreen interpretation: ${answer.toolUses.map(tool => tool.input?.visible_state || '').filter(Boolean).join('\n') || answer.text}`, toolUses: answer.toolUses, recentActions, intentNotes });
+      } catch (error) {
+        if (error instanceof ProxyError && error.isBudget) { budgetExceeded = true; quitReason = 'cost_cap'; quitDetail = error.message; break; }
+        if (error instanceof ProxyError && error.isClosed) { canceled = true; quitDetail = error.message; break; }
+        throw error;
+      }
+      transcript.write({ kind: 'action_selection', source: selection.source || 'jev', status: selection.status, confidence: selection.confidence, selected_tool_id: selection.toolId, reason: selection.reason });
+      if (selection.status !== 'selected') { quitReason = 'persona_quit'; quitDetail = 'Ravensight Playtester stopped for review: ' + selection.reason; break; }
+      if (selection.budget === 'warn') { budgetWarned = true; wrapUpLeft = Math.min(wrapUpLeft, WRAP_UP_TURNS); }
+    }
     messages.push({ role: 'assistant', content: assistantContent(answer) });
     const results = [];
     for (const toolUse of answer.toolUses) {
+      if (selection && toolUse.id !== selection.toolId) { results.push({ type: 'tool_result', tool_use_id: toolUse.id, content: 'Alternative not selected. No action executed.' }); continue; }
       const executed = await executeTool({
         driver,
         toolUse,
@@ -381,11 +458,11 @@ async function playLoop({
           // (bug, confusing, good, suggestion, reaction) would otherwise
           // overwrite the transcript line's `kind`, and every note would
           // read as a different sort of line.
-          notes.push({ ...note, at_action: actions });
+          notes.push({ ...note, at_action: actions, order: nextOrder() });
           transcript.write({ kind: 'note', tool: toolUse.name, note });
         },
         onScreenshot: shot => {
-          screenshots.push(shot);
+          screenshots.push({ ...shot, order: nextOrder() });
           transcript.write({ kind: 'screenshot', path: shot.relativePath, reason: shot.reason });
         }
       });
@@ -402,6 +479,10 @@ async function playLoop({
         actions
       });
       if (executed.observation) {
+        latestObservation = executed.observation.text;
+        visualObservation = Boolean(executed.observation.screenshotBase64);
+        recentActions.push(`${toolUse.name}: ${executed.changed ? 'screen changed' : 'no observed change'}`);
+        if (recentActions.length > 10) recentActions.shift();
         transcript.write({ kind: 'observation', step: executed.observation.step, text: executed.observation.text, url: executed.observation.url });
         // Only an input to the game can tell us anything about being stuck. A
         // `snapshot` or a `screenshot` never changes the screen by design, so
@@ -423,13 +504,17 @@ async function playLoop({
 
     const consoleErrors = typeof driver.consoleErrors === 'function' ? await driver.consoleErrors() : [];
     if (consoleErrors.length > 0) {
-      // An uncaught error in the game is a finding whether or not the
+      // An uncaught error in the game is worth recording whether or not the
       // persona noticed it, so it is logged and evidenced automatically
-      // (spec 03). It is the model's job to decide the severity.
+      // (spec 03). It carries no severity: the report step decides that,
+      // with the persona's view of the screen in front of it. A 4xx on a
+      // resource is a `network` note, which the report step reads as context
+      // rather than as a bug (a missing favicon is not a broken game).
       const shot = await safeScreenshot(driver, 'console-error', screenshots, transcript);
       for (const entry of consoleErrors) {
-        const note = { kind: 'bug', severity: 'major', text: `${entry.type}: ${entry.text}`, automatic: true };
-        notes.push(note);
+        const kind = RESOURCE_4XX.test(String(entry.text)) ? 'network' : 'bug';
+        const note = { kind, text: `${entry.type}: ${entry.text}`, automatic: true };
+        notes.push({ ...note, at_action: actions, order: nextOrder() });
         transcript.write({ kind: 'note', note });
       }
       asides.push(`The game logged ${consoleErrors.length} error(s): ${consoleErrors.map(entry => entry.text).join(' | ').slice(0, 800)}${shot ? ` A screenshot is saved at ${shot.relativePath}.` : ''}`);
@@ -517,6 +602,8 @@ async function reportPhase({
         'Notes you logged, in order:',
         notes.length > 0 ? notes.map((note, i) => `${i + 1}. [${note.kind}${note.severity ? `/${note.severity}` : ''}] ${note.text}`).join('\n') : '(none)',
         '',
+        'Notes marked automatic were recorded by the runner from the console, with no severity: decide one from what the persona saw. Network notes are context, not bugs: a failed request is a finding only when the persona saw a broken screen because of it.',
+        '',
         'Screenshots available as evidence (use these exact paths in evidence refs):',
         screenshots.length > 0 ? screenshots.map(shot => `- ${shot.relativePath}${shot.reason ? ` (${shot.reason})` : ''}`).join('\n') : '(none)',
         '',
@@ -642,7 +729,10 @@ async function reportPhase({
 async function callWithImageRetry({ imageBudget, keepObservations, messages, onImageBudget, ...rest }) {
   let budget = imageBudget;
   for (let attempt = 0; attempt < 3; attempt += 1) {
-    const prepared = prepareMessages(messages, { maxImages: budget, keepObservations });
+    // One moving cache breakpoint on the newest block, as the fix loop does:
+    // the prefix up to it is what the next turn re-sends, so the next turn
+    // reads it from the cache instead of paying for it again.
+    const prepared = withRollingCacheBreakpoint(prepareMessages(messages, { maxImages: budget, keepObservations }));
     try {
       return await callStepWithRetry({ ...rest, messages: prepared });
     } catch (error) {
@@ -758,7 +848,7 @@ function redactToolInput(toolUse) {
 async function safeScreenshot(driver, name, screenshots, transcript) {
   try {
     const shot = await driver.screenshot(name);
-    screenshots.push({ ...shot, reason: name });
+    screenshots.push({ ...shot, reason: name, order: nextOrder() });
     transcript.write({ kind: 'screenshot', path: shot.relativePath, reason: name });
     return shot;
   } catch (error) {
@@ -850,6 +940,104 @@ export function renderFallbackMarkdown({ persona, driverName, target, notes, scr
   ].join('\n');
 }
 
+/**
+ * The screenshot nearest a note, by the order they happened in; a tie goes
+ * to the later one, since a persona usually notes something and then
+ * captures it.
+ *
+ * @param {Object} note
+ * @param {Array<Object>} screenshots
+ * @returns {Object|null}
+ */
+export function nearestScreenshot(note, screenshots) {
+  let best = null;
+  let bestDistance = Infinity;
+  for (const shot of screenshots) {
+    if (!shot || !shot.relativePath || !Number.isFinite(shot.order) || !Number.isFinite(note.order)) continue;
+    const distance = Math.abs(shot.order - note.order);
+    if (distance < bestDistance || (distance === bestDistance && best && shot.order > best.order)) {
+      best = shot;
+      bestDistance = distance;
+    }
+  }
+  return best || screenshots.find(shot => shot && shot.relativePath) || null;
+}
+
+/**
+ * The persona's notes as report findings, for a run with no report step.
+ *
+ * Nothing is judged here, because there is no model turn to judge with:
+ * the persona's own kind becomes the category, its own severity is kept
+ * when it gave one and is `minor` otherwise (`praise` for a `good` note), and
+ * a network note is left out, since it is context rather than a finding.
+ *
+ * @param {Array<Object>} notes
+ * @param {Array<Object>} screenshots
+ * @returns {Array<Object>}
+ */
+export function notesToFindings(notes, screenshots) {
+  const findings = [];
+  for (const note of notes) {
+    const category = NOTE_CATEGORY[note.kind];
+    if (!category || !note.text) continue;
+    const text = String(note.text);
+    const shot = nearestScreenshot(note, screenshots);
+    findings.push({
+      id: `F${String(findings.length + 1).padStart(3, '0')}`,
+      severity: note.kind === 'good' ? 'praise' : (note.severity && note.severity !== 'praise' ? note.severity : 'minor'),
+      category,
+      title: text.length > 120 ? `${text.slice(0, 117)}...` : text,
+      description: text,
+      repro_steps: [`Seen after ${Number(note.at_action) || 0} action(s) in this run.`],
+      evidence: shot
+        ? [{ type: 'screenshot', ref: shot.relativePath }]
+        : [{ type: 'transcript_step', ref: `action ${Number(note.at_action) || 0}` }]
+    });
+  }
+  return findings;
+}
+
+/**
+ * The report for a run the money ran out on: the notes as findings,
+ * `partial: true`, and both files written. The sentiment is the scale's
+ * midpoint and `would_continue` is false, because the schema requires both
+ * and no persona verdict was recorded; `partial` is what tells a reader not
+ * to read them as one.
+ */
+async function writePartialReport({ outcome, runDirectory, persona, driverName, target, notes, screenshots, play, skill, ledger, runId, jobId, startedAt, endedAt }) {
+  const report = stampReport({
+    findings: notesToFindings(notes, screenshots),
+    sentiment: { fun: 3, clarity: 3, frustration: 3 },
+    would_continue: false,
+    partial: true
+  }, {
+    runId,
+    jobId,
+    persona: persona.slug,
+    driver: driverName,
+    target,
+    startedAt,
+    endedAt,
+    actionsTaken: play.actionsTaken,
+    usage: ledger.toReportUsage(),
+    promptVersion: promptVersionOf(skill) || 'unknown',
+    quitReason: 'cost_cap',
+    quitDetail: play.quitDetail
+  });
+  const markdown = renderFallbackMarkdown({
+    persona,
+    driverName,
+    target,
+    notes,
+    screenshots,
+    play,
+    reason: 'The run reached its model budget before the report step. What the persona saw is below, and report.json carries the notes as unreviewed findings.'
+  });
+  await writeReportFiles({ runDirectory, markdown, report });
+  outcome.report = report;
+  outcome.reportMarkdown = markdown;
+}
+
 async function writePartialMarkdown({ runDirectory, persona, driverName, target, notes, screenshots, play, reason }) {
   const markdown = renderFallbackMarkdown({ persona, driverName, target, notes, screenshots, play, reason });
   assertNoSecrets('report.md', markdown);
@@ -868,4 +1056,4 @@ export function promptVersionOf(skill) {
   return match ? match[1] : null;
 }
 
-export default { runPersona, buildSystem, promptVersionOf, shouldEscalate, renderFallbackMarkdown };
+export default { runPersona, buildSystem, promptVersionOf, shouldEscalate, renderFallbackMarkdown, notesToFindings, nearestScreenshot };

package/src/run/resume.js

@@ -153,6 +153,7 @@ export async function resumeJob({ api, gameId, jobId, job, runs, state, repoRoot
 
   const resumeOptions = {
     ...options,
+    controller: journal.options?.controller || options.controller || 'standard',
     maxActions: (journal.options && journal.options.maxActions) ?? options.maxActions,
     concurrency: options.concurrency ?? (journal.options && journal.options.concurrency),
     uploadVideo: options.uploadVideo ?? (journal.options && journal.options.uploadVideo),

package/src/run/tools.js

@@ -1,3 +1,4 @@
+import { UNCHANGED_OBSERVATION_TEXT } from './context.js';
 import { OBSERVATION_MAX_CHARS, truncateObservation } from './drivers/driver.js';
 
 /**
@@ -29,7 +30,16 @@ export const NOTE_KINDS = Object.freeze(['bug', 'confusing', 'good', 'suggestion
 export const SEVERITIES = Object.freeze(['blocker', 'major', 'minor', 'cosmetic', 'praise']);
 
 /** Tools that spend an action. */
-export const ACTION_TOOLS = Object.freeze(['click', 'type', 'key', 'navigate']);
+export const ACTION_TOOLS = Object.freeze(['click', 'type', 'key', 'navigate', 'swipe']);
+
+export const SWIPE_TOOL = {
+  name: 'swipe',
+  description: 'Drag from one screenshot coordinate to another. Spends one action.',
+  input_schema: {
+    type: 'object', additionalProperties: false, required: ['x', 'y', 'toX', 'toY'],
+    properties: Object.fromEntries(['x', 'y', 'toX', 'toY', 'durationMs'].map(key => [key, { type: 'number', minimum: 0, maximum: key === 'durationMs' ? 2000 : 32768 }]))
+  }
+};
 
 const thought = {
   type: 'string',
@@ -185,6 +195,8 @@ export function actionFor(toolUse) {
       return { kind: 'navigate', url: String(input.url || '') };
     case 'wait':
       return { kind: 'wait', ms: Number(input.ms) || 0 };
+    case 'swipe':
+      return { kind: 'swipe', from: { x: input.x, y: input.y }, to: { x: input.toX, y: input.toY }, durationMs: input.durationMs };
     default:
       return null;
   }
@@ -289,13 +301,17 @@ export async function executeTool({ driver, toolUse, onNote, onScreenshot }) {
  * @param {boolean} [isError]
  */
 export function observationResult(id, observation, prefix = '', isError = false) {
+  // The same screen as last time is a marker, not a resend: the persona
+  // already has it in context, and a stuck run would otherwise pay for the
+  // whole snapshot (and its image) on every turn it stays stuck.
+  const body = observation.unchanged ? UNCHANGED_OBSERVATION_TEXT : truncateObservation(observation.text, OBSERVATION_MAX_CHARS);
   const content = [
     {
       type: 'text',
-      text: `${prefix}step ${observation.step}${observation.url ? ` (${observation.url})` : ''}${citationLine(observation)}\n${truncateObservation(observation.text, OBSERVATION_MAX_CHARS)}`
+      text: `${prefix}step ${observation.step}${observation.url ? ` (${observation.url})` : ''}${citationLine(observation)}\n${body}`
     }
   ];
-  if (observation.screenshotBase64) {
+  if (observation.screenshotBase64 && !observation.unchanged) {
     content.push({
       type: 'image',
       source: {

package/src/states.js

@@ -26,7 +26,10 @@ export const DRIVERS = Object.freeze([
   'godot_driver',
   'cli_stdio',
   'unity_driver',
-  'unreal_driver'
+  'unreal_driver',
+  'desktop_driver',
+  'android_driver',
+  'ios_driver'
 ]);
 
 export const JOB_STATES = Object.freeze([

package/src/ui/index.js

@@ -157,6 +157,33 @@ export async function confirm(question, options = {}) {
   }
 }
 
+/**
+ * Ask the person to pick one of a numbered list. Like `confirm`, it never
+ * asks without a terminal: with no TTY (CI, a pipe) it answers null at once,
+ * and so does an answer that is not one of the numbers shown.
+ *
+ * @param {string} question e.g. "Which game? [1-3]"
+ * @param {Array<string>} labels one line per choice, printed numbered from 1
+ * @param {{input?: object, output?: object}} [options]
+ * @returns {Promise<number|null>} the zero based index, or null
+ */
+export async function choose(question, labels, options = {}) {
+  const input = options.input || process.stdin;
+  if (!input.isTTY) return null;
+  const output = options.output || stream;
+
+  labels.forEach((label, index) => output.write(`  ${index + 1}. ${label}\n`));
+  const rl = createInterface({ input, output });
+  try {
+    const answer = (await rl.question(`${question} `)).trim();
+    if (!/^\d+$/.test(answer)) return null;
+    const index = Number(answer) - 1;
+    return index >= 0 && index < labels.length ? index : null;
+  } finally {
+    rl.close();
+  }
+}
+
 /** Cents as dollars, the one place that formatting exists. */
 export function money(cents) {
   if (cents === null || cents === undefined) return 'unknown';
@@ -190,6 +217,7 @@ export const ui = {
   table,
   spinner,
   confirm,
+  choose,
   money,
   bytes
 };