ravensight-playtest 0.7.0 → 0.7.5

package/src/run/report.js

@@ -57,6 +57,60 @@ export function lintMarkdownSections(markdown) {
   return REQUIRED_MARKDOWN_SECTIONS.filter(section => !text.includes(section));
 }
 
+/** The tool-call tags a model has been seen to leak into a written file. */
+const TOOL_TAG = '(?:parameter|invoke|content|function_calls|function_results|tool_use|tool_call|antml:[\\w-]+)';
+
+/** A line made only of tags: closers of any name, or tool-call openers. */
+const MARKUP_LINE = new RegExp(`^(?:\\s*(?:<\\/[A-Za-z][\\w:.-]*\\s*>|<${TOOL_TAG}\\b[^>]*>))+\\s*$`);
+
+/** Tool-call closers glued to the end of the last line of real text. */
+const MARKUP_SUFFIX = new RegExp(`(?:\\s*<\\/${TOOL_TAG}\\s*>)+\\s*$`);
+
+const FENCE = /^\s*(?:```|~~~)/;
+
+/**
+ * Takes tool-call markup off the end of a file the report step wrote. A real
+ * report.md ended "</content>\n</invoke>" because the model closed its tool
+ * call inside the file's content, and the runner appended its own sections
+ * after the stray tags.
+ *
+ * Only the end of the text is touched, and only outside a code fence: HTML a
+ * report quotes in backticks or in a fenced block is kept.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+export function stripTrailingToolMarkup(text) {
+  if (typeof text !== 'string' || text === '') return text;
+  const lines = text.split('\n');
+  const insideFence = count => lines.slice(0, count).filter(line => FENCE.test(line)).length % 2 === 1;
+  let end = lines.length;
+  let changed = false;
+  while (end > 0) {
+    const line = lines[end - 1];
+    if (line.trim() === '') {
+      end -= 1;
+    } else if (MARKUP_LINE.test(line) && !insideFence(end - 1)) {
+      end -= 1;
+      changed = true;
+    } else {
+      break;
+    }
+  }
+  const kept = lines.slice(0, end);
+  if (kept.length > 0) {
+    const index = kept.length - 1;
+    const last = kept[index];
+    const backticks = (last.match(/`/g) || []).length;
+    if (MARKUP_SUFFIX.test(last) && backticks % 2 === 0 && !FENCE.test(last) && !insideFence(index)) {
+      kept[index] = last.replace(MARKUP_SUFFIX, '');
+      changed = true;
+    }
+  }
+  if (!changed) return text;
+  return `${kept.join('\n').replace(/\s+$/, '')}\n`;
+}
+
 export class ReportInvalidError extends Error {
   /**
    * @param {Array<{instancePath: string, message: string}>} details

package/src/run/tools.js

@@ -335,6 +335,19 @@ export async function executeTool({ driver, toolUse, onNote, onScreenshot, stale
     : action.ref && staleRefs
       ? { ok: false, changed: false, error: STALE_REF_MESSAGE, driverError: true }
       : await driver.act(action);
+  // The runner refused to leave the build. The persona is told so in plain
+  // text and plays on: it is a step that changed nothing, never an error and
+  // never a run failure. No observation either, because nothing moved.
+  if (outcome.refused === 'off_site') {
+    return {
+      result: toolResult(toolUse.id, offSiteRefusalText(outcome.host)),
+      observation: null,
+      spentAction: ACTION_TOOLS.includes(toolUse.name),
+      changed: false,
+      error: null,
+      refused: 'off_site'
+    };
+  }
   const observation = await driver.observe();
   const spentAction = ACTION_TOOLS.includes(toolUse.name) && outcome.ok;
   const prefix = outcome.ok
@@ -350,11 +363,25 @@ export async function executeTool({ driver, toolUse, onNote, onScreenshot, stale
   };
 }
 
+/**
+ * What the persona is told when the runner refuses to leave the build.
+ *
+ * @param {string} [host]
+ * @returns {string}
+ */
+export function offSiteRefusalText(host) {
+  return `Refused: that link leaves the build under test (${host || 'another site'}). Stay on this page and use its controls.`;
+}
+
 /**
  * A tool_result whose content is the new observation, with the screenshot
  * attached as an image when the driver decided the text was too thin to
  * play from. Images are pruned from history by the caller: the act step
- * accepts a very small number of image blocks per request.
+ * accepts a very small number of image blocks per request. When the screenshot
+ * rides along, a failed action is NOT flagged `is_error`: the Messages API
+ * refuses an `is_error` result that carries an image, and the image is the
+ * persona's only view of a canvas game. The "That did not work" prefix
+ * already says what the flag would.
  *
  * @param {string} id
  * @param {Object} observation
@@ -382,7 +409,8 @@ export function observationResult(id, observation, prefix = '', isError = false)
       }
     });
   }
-  return { type: 'tool_result', tool_use_id: id, content, ...(isError ? { is_error: true } : {}) };
+  const withImage = content.length > 1;
+  return { type: 'tool_result', tool_use_id: id, content, ...(isError && !withImage ? { is_error: true } : {}) };
 }
 
 /**
@@ -429,4 +457,4 @@ function message(error) {
   return String(error && error.message ? error.message : error);
 }
 
-export default { PLAY_TOOLS, REPORT_TOOLS, executeTool, actionFor, toolResult, observationResult };
+export default { PLAY_TOOLS, REPORT_TOOLS, executeTool, actionFor, toolResult, observationResult, offSiteRefusalText };

package/src/run/usage.js

@@ -32,6 +32,14 @@ export function createUsageLedger() {
   let modelMs = 0;
   let serverTotalUsd = null;
   let serverCalls = null;
+  // The run's tokens in input-token units (see weightedUnits), and the last
+  // server spend that covered every call this ledger had recorded when it
+  // arrived. Together they say what one unit costs on this run, which is how
+  // the persona loop keeps enough budget back for the report without ever
+  // holding a price: ./budgetReserve.js.
+  let units = 0;
+  let lastCallUnits = 0;
+  let sample = null;
 
   return {
     /**
@@ -50,6 +58,8 @@ export function createUsageLedger() {
       entry.cache_read_tokens += Number(usage.cache_read_input_tokens || 0);
       entry.output_tokens += Number(usage.output_tokens || 0);
       byModel.set(key, entry);
+      lastCallUnits = weightedUnits(usage);
+      units += lastCallUnits;
     },
 
     /**
@@ -61,6 +71,31 @@ export function createUsageLedger() {
       if (!runUsage) return;
       if (Number.isFinite(Number(runUsage.total_usd))) serverTotalUsd = Number(runUsage.total_usd);
       if (Number.isFinite(Number(runUsage.calls))) serverCalls = Number(runUsage.calls);
+      // A figure that has not caught up with every call recorded here would
+      // make a unit look cheaper than it is, so only one that covers them all
+      // (the server may count more, which only makes a unit look dearer) is
+      // kept as the price sample.
+      const covers = !Number.isFinite(Number(runUsage.calls)) || Number(runUsage.calls) >= calls;
+      if (serverTotalUsd !== null && serverTotalUsd > 0 && units > 0 && covers) sample = { usd: serverTotalUsd, units };
+    },
+
+    /** The weighted units of the last call recorded, 0 before the first. */
+    get lastCallUnits() {
+      return lastCallUnits;
+    },
+
+    /**
+     * What the run has spent and what one weighted unit costs on it, or null
+     * before the server has priced a call this ledger recorded. The spend is
+     * the server's figure plus the calls made since, priced at the sample's
+     * rate, so it never reads lower than the server's own.
+     *
+     * @returns {{usd: number, usdPerUnit: number}|null}
+     */
+    spendEstimate() {
+      if (!sample) return null;
+      const usdPerUnit = sample.usd / sample.units;
+      return { usd: Math.max(serverTotalUsd || 0, sample.usd + usdPerUnit * Math.max(0, units - sample.units)), usdPerUnit };
     },
 
     get modelCalls() {
@@ -114,6 +149,25 @@ export function createUsageLedger() {
   };
 }
 
+/** Input-token units per cache write, cache read and output token. */
+export const CACHE_WRITE_WEIGHT = 1.25;
+export const CACHE_READ_WEIGHT = 0.1;
+export const OUTPUT_WEIGHT = 5;
+
+/**
+ * One call's tokens in input-token units: what the server's price for the call
+ * is proportional to on every tier, since each tier prices a cache write at
+ * 1.25 input tokens, a cache read at 0.1 and an output token at 5. No price,
+ * only the shape of one.
+ *
+ * @param {Object} usage
+ * @returns {number}
+ */
+export function weightedUnits(usage = {}) {
+  const n = key => Math.max(0, Number(usage && usage[key]) || 0);
+  return n('input_tokens') + n('cache_creation_input_tokens') * CACHE_WRITE_WEIGHT + n('cache_read_input_tokens') * CACHE_READ_WEIGHT + n('output_tokens') * OUTPUT_WEIGHT;
+}
+
 /**
  * @param {string} runDirectory
  * @param {ReturnType<typeof createUsageLedger>} ledger

package/src/ui/index.js

@@ -1,5 +1,6 @@
 import { createInterface } from 'node:readline/promises';
 import { stableStringify } from '../fsutil.js';
+import { CliError, ExitCode } from '../errors.js';
 
 /**
  * Terminal output, in one place.
@@ -183,22 +184,36 @@ export function spinner(label) {
   };
 }
 
+/** The one line printed when a question needs a terminal and there is none. */
+export const NO_TERMINAL_MESSAGE = 'No terminal to confirm in: pass --yes to accept the shown price and hold.';
+
+function noTerminal(message = NO_TERMINAL_MESSAGE) {
+  const error = new CliError(message, ExitCode.FAILURE);
+  error.code = 'no_terminal';
+  return error;
+}
+
+/** True for the error confirm() throws when there is no terminal to ask in. */
+export const isNoTerminal = error => error instanceof CliError && error.code === 'no_terminal';
+
 /**
  * Ask a yes or no question.
  *
- * `--yes` skips it, and so does a non interactive stdin: a prompt nobody can
- * answer would hang a CI job forever, so the absence of a TTY is treated as a
- * refusal rather than as an assumed yes. That is the safe direction for a
- * question that is usually "shall I spend your money".
+ * `--yes` skips it. A non interactive stdin is an error, not an answer: a
+ * prompt nobody can answer would hang a CI job forever, and piped text such as
+ * `echo y |` is never taken as consent, because the question is usually
+ * "shall I spend your money". Quietly declining instead left people wondering
+ * why nothing happened, so the error says what to do.
  *
  * @param {string} question
- * @param {{yes?: boolean, input?: object, output?: object}} [options]
+ * @param {{yes?: boolean, input?: object, output?: object, noTerminalMessage?: string}} [options]
+ *   noTerminalMessage replaces the line for a question that is not about money.
  * @returns {Promise<boolean>}
  */
 export async function confirm(question, options = {}) {
   if (options.yes) return true;
   const input = options.input || process.stdin;
-  if (!input.isTTY && !options.input) return false;
+  if (!input.isTTY && !options.input) throw noTerminal(options.noTerminalMessage);
 
   const rl = createInterface({ input, output: options.output || stream });
   try {