ravensight-playtest 0.2.8 → 0.3.0

package/src/run/regenerate.js

@@ -47,7 +47,7 @@ export async function readTranscript(runDirectory) {
   try {
     text = await readFile(path.join(runDirectory, RUN_FILES.transcript), 'utf8');
   } catch {
-    return { header: null, notes: [], screenshots: [], observations: 0, actions: 0 };
+    return { header: null, notes: [], screenshots: [], observations: 0, actions: 0, firstT: null, lastT: null };
   }
 
   let header = null;
@@ -55,6 +55,12 @@ export async function readTranscript(runDirectory) {
   const screenshots = [];
   let observations = 0;
   let actions = 0;
+  // Notes written on the tool call the last `action` line closed, so a
+  // `driver_error` after it can mark them as the live run did.
+  let callNotes = [];
+  let openNotes = [];
+  let firstT = null;
+  let lastT = null;
   for (const line of text.split('\n')) {
     if (!line.trim()) continue;
     let entry;
@@ -65,13 +71,27 @@ export async function readTranscript(runDirectory) {
       // it rather than refusing to read the whole transcript.
       continue;
     }
+    if (Number.isFinite(Number(entry.t))) {
+      if (firstT === null) firstT = Number(entry.t);
+      lastT = Number(entry.t);
+    }
     if (entry.kind === 'header') header = entry;
-    else if (entry.kind === 'note' && entry.note) notes.push(entry.note);
+    else if (entry.kind === 'note' && entry.note) {
+      const note = { ...entry.note };
+      notes.push(note);
+      if (entry.tool) openNotes.push(note);
+    } else if (entry.kind === 'driver_error') {
+      for (const note of callNotes) note.driver_error = true;
+    }
     else if (entry.kind === 'screenshot' && entry.path) screenshots.push({ relativePath: entry.path, reason: entry.reason || '' });
     else if (entry.kind === 'observation') observations += 1;
-    else if (entry.kind === 'action' && Number.isFinite(Number(entry.actions))) actions = Math.max(actions, Number(entry.actions));
+    else if (entry.kind === 'action') {
+      callNotes = openNotes;
+      openNotes = [];
+      if (Number.isFinite(Number(entry.actions))) actions = Math.max(actions, Number(entry.actions));
+    }
   }
-  return { header, notes, screenshots, observations, actions };
+  return { header, notes, screenshots, observations, actions, firstT, lastT };
 }
 
 /**

package/src/run/report.js

@@ -127,6 +127,86 @@ export function stampReport(report, facts) {
   return stamped;
 }
 
+/** Keys a model has been seen to put an evidence path under instead of `ref`. */
+const REF_ALIASES = ['path', 'file', 'screenshot', 'src', 'href', 'location', 'value'];
+
+/** The four evidence types the schema allows. */
+const EVIDENCE_TYPES = new Set(['screenshot', 'video_ts', 'transcript_step', 'file_line']);
+
+/**
+ * One evidence entry in the schema's shape, `{type, ref}` and nothing else,
+ * when the model's own words say unambiguously what it meant.
+ *
+ * The 0.2.8 report that failed upload wrote
+ * `{"type": "screenshot", "path": "screenshots/04-04-school-selected.png"}`
+ * five times: a real file, under the wrong key. The schema refuses the extra
+ * key and the missing `ref`, so the whole report was refused and the run
+ * failed over a key name. This moves the value, never invents one: a path
+ * still has to name a file in the run directory (`checkEvidence`), and the
+ * server still checks it against what was uploaded.
+ *
+ * Also: a bare string becomes a screenshot or a transcript step by its look;
+ * a missing type on a `screenshots/...` ref becomes `screenshot`; a ref
+ * written `./screenshots/x.png` or as an absolute path inside the run
+ * directory becomes the run relative path the server's manifest uses.
+ *
+ * @param {any} item
+ * @param {string|null} runDirectory
+ * @returns {any} the normalised entry, or the input unchanged when it is not an object it can read
+ */
+export function normaliseEvidenceItem(item, runDirectory = null) {
+  if (typeof item === 'string') {
+    const text = item.trim();
+    if (/^(?:\.\/)?screenshots\//.test(text)) return normaliseEvidenceItem({ type: 'screenshot', ref: text }, runDirectory);
+    if (/^(?:observation|note|step|action)\s*#?\s*\d+$/i.test(text)) return { type: 'transcript_step', ref: text };
+    return item;
+  }
+  if (!item || typeof item !== 'object' || Array.isArray(item)) return item;
+  let ref = typeof item.ref === 'string' && item.ref.trim() ? item.ref : null;
+  if (ref === null) {
+    const alias = REF_ALIASES.find(key => typeof item[key] === 'string' && item[key].trim());
+    if (alias) ref = item[alias];
+  }
+  let type = typeof item.type === 'string' ? item.type : null;
+  if (!type && ref && /^(?:\.\/)?screenshots\//.test(ref.trim())) type = 'screenshot';
+  if (type === null || ref === null || !EVIDENCE_TYPES.has(type)) return item;
+  ref = ref.trim();
+  if (type === 'screenshot') ref = runRelative(ref, runDirectory);
+  return { type, ref };
+}
+
+function runRelative(ref, runDirectory) {
+  let value = ref.replace(/\\/g, '/');
+  if (runDirectory && path.isAbsolute(ref)) {
+    const relative = path.relative(path.resolve(runDirectory), path.resolve(ref));
+    if (relative && !relative.startsWith('..') && !path.isAbsolute(relative)) value = relative.split(path.sep).join('/');
+  }
+  while (value.startsWith('./')) value = value.slice(2);
+  return value;
+}
+
+/**
+ * Every finding's evidence through {@link normaliseEvidenceItem}.
+ *
+ * @param {Object} report
+ * @param {string|null} [runDirectory]
+ * @returns {{report: Object, changed: number}} a new report, and how many entries were rewritten
+ */
+export function normaliseEvidence(report, runDirectory = null) {
+  if (!report || typeof report !== 'object' || !Array.isArray(report.findings)) return { report, changed: 0 };
+  let changed = 0;
+  const findings = report.findings.map(finding => {
+    if (!finding || typeof finding !== 'object' || !Array.isArray(finding.evidence)) return finding;
+    const evidence = finding.evidence.map(item => {
+      const out = normaliseEvidenceItem(item, runDirectory);
+      if (JSON.stringify(out) !== JSON.stringify(item)) changed += 1;
+      return out;
+    });
+    return { ...finding, evidence };
+  });
+  return { report: { ...report, findings }, changed };
+}
+
 /**
  * Check every `screenshot` evidence ref against the files actually on disk.
  *
@@ -151,8 +231,17 @@ export async function checkEvidence(report, runDirectory) {
       continue;
     }
     for (const item of evidence) {
-      if (!item || item.type !== 'screenshot') continue;
-      const ref = String(item.ref || '');
+      if (!item || typeof item !== 'object') {
+        problems.push(`${finding.id}: an evidence entry is not an object; write {"type": ..., "ref": ...}`);
+        continue;
+      }
+      if (typeof item.ref !== 'string' || !item.ref.trim()) {
+        const keys = Object.keys(item).filter(key => key !== 'type');
+        problems.push(`${finding.id}: a ${item.type || 'typeless'} evidence entry has no "ref"${keys.length > 0 ? ` (it has ${keys.map(key => `"${key}"`).join(', ')})` : ''}; every entry is exactly {"type": ..., "ref": ...}`);
+        continue;
+      }
+      if (item.type !== 'screenshot') continue;
+      const ref = String(item.ref);
       const candidate = path.resolve(runDirectory, ref);
       // Refuse a ref that climbs out of the run directory before touching
       // the filesystem with it.
@@ -214,6 +303,8 @@ export default {
   createReportValidator,
   stampReport,
   checkEvidence,
+  normaliseEvidence,
+  normaliseEvidenceItem,
   writeReportFiles,
   repairInstruction,
   lintMarkdownSections,

package/src/run/resume.js

@@ -62,6 +62,15 @@ export async function resumeJob({ api, gameId, jobId, job, runs, state, repoRoot
     return EXIT.ENVIRONMENT;
   }
 
+  // A run whose report the server refused is done here, whatever the server
+  // says its state is: it is never finalised from its transcript (which would
+  // rewrite the kept report.json and report.md) and never replayed.
+  for (const [runId, entry] of Object.entries(journal.runs || {})) {
+    if (entry && entry.report_refused === true) {
+      log(`run ${runId}: the server refused its report, so it is not replayed or rebuilt. Its report.json, report.md and screenshots are in ${runDirFor(repoRoot, jobId, runId)}.`);
+    }
+  }
+
   // Reconcile the journal with what the server actually thinks: a run the sweep
   // marked interrupted, or one that finished after the journal was last
   // written, is the server's word against a stale file, and the server wins.
@@ -235,7 +244,13 @@ async function finalizeFromTranscript({ deps, api, gameId, jobId, repoRoot, row,
         actions_taken: report.actions_taken,
         quit_reason: 'error',
         report,
-        report_md: markdown
+        report_md: markdown,
+        // The transcript's own span: its first and last timestamps. No model
+        // time is known for a run rebuilt after the process died.
+        duration: {
+          wall_seconds: transcript.firstT !== null && transcript.lastT !== null ? Number(((transcript.lastT - transcript.firstT) / 1000).toFixed(3)) : null,
+          model_seconds: null
+        }
       }
     });
     if (answer && answer.dropped_findings > 0) {

package/src/run/state.js

@@ -192,7 +192,10 @@ export async function patchRun(repoRoot, state, runId, patch) {
  * @returns {boolean}
  */
 export function isRunDone(entry) {
-  return Boolean(entry) && (entry.completed === true || RUN_DONE_STATES.includes(entry.state));
+  // A run whose report the server refused was played to the end: replaying
+  // it would spend a unit again, and rebuilding its report from the
+  // transcript would overwrite the one the developer is meant to read.
+  return Boolean(entry) && (entry.completed === true || entry.report_refused === true || RUN_DONE_STATES.includes(entry.state));
 }
 
 export default { SCHEMA_VERSION, newState, newRunEntry, save, load, patchRun, isRunDone, stateFile };

package/src/run/tools.js

@@ -1,5 +1,7 @@
 import { UNCHANGED_OBSERVATION_TEXT } from './context.js';
-import { OBSERVATION_MAX_CHARS, truncateObservation } from './drivers/driver.js';
+import { OBSERVATION_MAX_CHARS, STALE_REF_MESSAGE, truncateObservation } from './drivers/driver.js';
+
+export { STALE_REF_MESSAGE };
 
 /**
  * The client tool set for `persona_playtest.act`.
@@ -23,8 +25,12 @@ import { OBSERVATION_MAX_CHARS, truncateObservation } from './drivers/driver.js'
  * `snapshot`, `screenshot` and `wait` do not.
  */
 
-/** Note kinds, from spec 03's `log_note`. */
-export const NOTE_KINDS = Object.freeze(['bug', 'confusing', 'good', 'suggestion', 'reaction']);
+/**
+ * Note kinds, from spec 03's `log_note`, plus `waiting`: the persona is
+ * waiting for something to finish (a load, an animation, a turn). A waiting
+ * note is context for the report and never becomes a finding.
+ */
+export const NOTE_KINDS = Object.freeze(['bug', 'confusing', 'good', 'suggestion', 'reaction', 'waiting']);
 
 /** Severities, from the report schema's finding severity enum. */
 export const SEVERITIES = Object.freeze(['blocker', 'major', 'minor', 'cosmetic', 'praise']);
@@ -52,7 +58,7 @@ const note = {
   additionalProperties: false,
   required: ['kind', 'text'],
   properties: {
-    kind: { type: 'string', enum: [...NOTE_KINDS] },
+    kind: { type: 'string', enum: [...NOTE_KINDS], description: 'waiting means you are waiting for something to finish; it is recorded as context and never reported as a finding.' },
     text: { type: 'string' },
     severity: { type: 'string', enum: [...SEVERITIES] }
   }
@@ -71,12 +77,13 @@ export const PLAY_TOOLS = Object.freeze([
   },
   {
     name: 'click',
-    description: 'Click one thing. Spends an action. Either name the target or give x and y pixel coordinates. What a named target means depends on the driver: on playwright_web it is an accessibility node such as button "Start", a visible text fragment, or a CSS selector; on godot_driver it is a node path from the scene tree you were shown.',
+    description: 'Click one thing. Spends an action. On playwright_web, use the ref the screen shows next to the element, such as e12: it names exactly one element, even when two share a name. Otherwise name the target: on playwright_web an accessibility node such as button "Start" or a visible text fragment; on godot_driver a node path from the scene tree you were shown. Targets are refs or visible text, never locator code such as getByRole(...) or page.locator(...). Avoid x and y coordinates on playwright_web: they are blind to the page, and are only for a canvas with no refs, taken from the @(x,y) the screen shows or from the screenshot.',
     input_schema: {
       type: 'object',
       additionalProperties: false,
       properties: {
-        target: { type: 'string', description: 'web: button "Start", Start Game, or #start-button. Godot: a node path such as /root/Main/UI/StartButton.' },
+        ref: { type: 'string', description: 'playwright_web only, the preferred target: the ref shown on the element, such as e12.' },
+        target: { type: 'string', description: 'web: button "Start" or Start Game. Godot: a node path such as /root/Main/UI/StartButton.' },
         x: { type: 'number' },
         y: { type: 'number' },
         thought_in_character: thought,
@@ -95,7 +102,8 @@ export const PLAY_TOOLS = Object.freeze([
         text: { type: 'string' },
         submit: { type: 'boolean' },
         replace: { type: 'boolean', description: 'Empty the field first, so it holds only this text. playwright_web only; other drivers refuse it.' },
-        target: { type: 'string', description: 'playwright_web only, other drivers refuse it: the field to type into, named like a click target, such as textbox "Email". Without it, the focused field.' },
+        ref: { type: 'string', description: 'playwright_web only, the preferred way to name the field: its ref, such as e12.' },
+        target: { type: 'string', description: 'playwright_web only, other drivers refuse it: the field to type into, named like a click target, such as textbox "Email". Without it or a ref, the focused field.' },
         thought_in_character: thought,
         note
       }
@@ -186,6 +194,7 @@ export function actionFor(toolUse) {
   const input = toolUse.input || {};
   switch (toolUse.name) {
     case 'click':
+      if (typeof input.ref === 'string' && input.ref.trim()) return { kind: 'click', ref: input.ref.trim() };
       return Number.isFinite(Number(input.x)) && Number.isFinite(Number(input.y))
         ? { kind: 'click', target: { x: Number(input.x), y: Number(input.y) } }
         : { kind: 'click', target: String(input.target || '') };
@@ -193,6 +202,7 @@ export function actionFor(toolUse) {
       const action = { kind: 'type', text: String(input.text || ''), submit: Boolean(input.submit) };
       if (input.replace === true) action.replace = true;
       if (typeof input.target === 'string' && input.target.trim()) action.target = input.target;
+      if (typeof input.ref === 'string' && input.ref.trim()) action.ref = input.ref.trim();
       return action;
     }
     case 'key':
@@ -220,9 +230,11 @@ export function actionFor(toolUse) {
  * @param {Object} args.toolUse
  * @param {(note: Object) => void} [args.onNote]
  * @param {(shot: Object) => void} [args.onScreenshot]
+ * @param {boolean} [args.staleRefs] an earlier action in this turn changed the
+ *   screen, so a ref in this call was read off a screen that is gone
  * @returns {Promise<{result: Object, observation: Object|null, spentAction: boolean, changed: boolean, error: string|null}>}
  */
-export async function executeTool({ driver, toolUse, onNote, onScreenshot }) {
+export async function executeTool({ driver, toolUse, onNote, onScreenshot, staleRefs = false }) {
   const input = toolUse.input || {};
   if (toolUse.malformed) {
     return {
@@ -280,7 +292,15 @@ export async function executeTool({ driver, toolUse, onNote, onScreenshot }) {
     };
   }
 
-  const outcome = await driver.act(action);
+  // A ref is a playwright_web target. Every other driver is told so by name,
+  // before anything reaches the game.
+  // A ref in a later tool call of a turn whose earlier call already changed
+  // the screen was read off a screen that is gone.
+  const outcome = action.ref && !driver.supportsRefs
+    ? { ok: false, changed: false, error: `ref targets are only available on playwright_web; ${driver.name || 'this driver'} takes a named target or coordinates`, driverError: true }
+    : action.ref && staleRefs
+      ? { ok: false, changed: false, error: STALE_REF_MESSAGE, driverError: true }
+      : await driver.act(action);
   const observation = await driver.observe();
   const spentAction = ACTION_TOOLS.includes(toolUse.name) && outcome.ok;
   const prefix = outcome.ok
@@ -291,7 +311,8 @@ export async function executeTool({ driver, toolUse, onNote, onScreenshot }) {
     observation,
     spentAction,
     changed: Boolean(outcome.changed),
-    error: outcome.ok ? null : String(outcome.error || 'act failed')
+    error: outcome.ok ? null : String(outcome.error || 'act failed'),
+    driverError: !outcome.ok && Boolean(outcome.driverError)
   };
 }
 

package/src/run/usage.js

@@ -29,6 +29,7 @@ export function createUsageLedger() {
   const byModel = new Map();
   const started = Date.now();
   let calls = 0;
+  let modelMs = 0;
   let serverTotalUsd = null;
   let serverCalls = null;
 
@@ -36,8 +37,10 @@ export function createUsageLedger() {
     /**
      * @param {{model: string|null, usage: Object}} turn
      */
-    record({ model, usage }) {
+    record({ model, usage, ms }) {
       calls += 1;
+      // Time spent waiting on the model, when the caller measured it.
+      if (Number.isFinite(Number(ms)) && Number(ms) > 0) modelMs += Number(ms);
       const key = model || 'unknown';
       const entry = byModel.get(key) || { model: key, input_tokens: 0, cache_read_tokens: 0, output_tokens: 0, usd: 0 };
       // The report schema has no cache-creation field, so creation tokens
@@ -64,6 +67,20 @@ export function createUsageLedger() {
       return calls;
     },
 
+    /**
+     * Wall time since the ledger was made, and time spent waiting on model
+     * calls, in seconds. What `complete` sends as `duration`, so the run card
+     * does not read 0s for a run that played for minutes.
+     *
+     * @returns {{wall_seconds: number, model_seconds: number}}
+     */
+    duration() {
+      return {
+        wall_seconds: Number(((Date.now() - started) / 1000).toFixed(3)),
+        model_seconds: Number((modelMs / 1000).toFixed(3))
+      };
+    },
+
     /**
      * The `usage` object as the report schema wants it.
      *
@@ -87,6 +104,7 @@ export function createUsageLedger() {
       const report = this.toReportUsage();
       return {
         ...report,
+        model_seconds: Number((modelMs / 1000).toFixed(3)),
         local_model_calls: calls,
         server_model_calls: serverCalls,
         usd_source: serverTotalUsd === null ? 'unavailable' : 'server',